shubhampachori12110095 / knowledge-distillation Goto Github PK

DROP DATABASE IF EXISTS `ifeng`;
CREATE DATABASE `ifeng`;
CREATE USER 'ifeng'@'%' IDENTIFIED BY '12345';
GRANT ALL PRIVILEGES ON  ifeng.* TO 'ifeng'@'%';
FLUSH PRIVILEGES;

USE `ifeng`;

CREATE TABLE `attr` (
  `pageId` int(11) NOT NULL DEFAULT '0',
  `name` varchar(255) NOT NULL DEFAULT '',
  `value` mediumtext,
  `src` tinyint(4) DEFAULT NULL,
  PRIMARY KEY (`pageId`,`name`),
  KEY `pageId` (`pageId`),
  KEY `srcIdx` (`src`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;


CREATE TABLE `linkattr` (
  `blockId` int(11) NOT NULL DEFAULT '0',
  `pageId` int(11) NOT NULL DEFAULT '0',
  `name` varchar(255) NOT NULL DEFAULT '',
  `value` mediumtext,
  PRIMARY KEY (`blockId`,`pageId`,`name`),
  KEY `linkIdx` (`blockId`,`pageId`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

#
# Source for table block
#

CREATE TABLE `block` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `tags` varchar(255) DEFAULT NULL,
  `lastVisitTime` datetime DEFAULT NULL,
  `lastFinishTime` datetime DEFAULT NULL,
  `updateInfo` varchar(255) DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;


#
# Source for table link
#

CREATE TABLE `link` (
  `blockId` int(11) NOT NULL DEFAULT '0',
  `pageId` int(11) NOT NULL DEFAULT '0',
  `pos` int(11) NOT NULL DEFAULT '0',
  `linkText` varchar(255) DEFAULT NULL,
  PRIMARY KEY (`blockId`,`pageId`),
  KEY `pageId` (`pageId`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

#
# Source for table page_block
#

CREATE TABLE `page_block` (
  `pageId` int(11) NOT NULL DEFAULT '0',
  `blockId` int(11) NOT NULL DEFAULT '0',
  `pos` smallint(6) NOT NULL DEFAULT '0',
  PRIMARY KEY (`pageId`,`pos`,`blockId`),
  KEY `blockId` (`blockId`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

#
# Source for table todolist
#

CREATE TABLE `todolist` (
  `siteId` int(11) NOT NULL DEFAULT '0',
  `pageId` int(11) NOT NULL DEFAULT '0',
  `url` varchar(2048) NOT NULL DEFAULT '',
  PRIMARY KEY (`siteId`,`pageId`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

CREATE TABLE `unfinished` (
  `id` int(11) NOT NULL,
  `url` varchar(1024) NOT NULL DEFAULT '',
  `depth` int NOT NULL,
  `priority` int NOT NULL DEFAULT '0',
  `failcount` int NOT NULL DEFAULT 0,
  `lastVisit` bigint,
  `redirectedUrl` varchar(1024),
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

#
# Source for table webpage
#

CREATE TABLE `bad_url` (
  `url` varchar(1024) NOT NULL DEFAULT '',
  `md5` char(32) NOT NULL,
  `lastUpdate` datetime DEFAULT NULL,
  PRIMARY KEY (`md5`)	
) ENGINE=InnoDB  DEFAULT CHARSET=utf8;

CREATE TABLE `webpage` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `url` varchar(1024) NOT NULL DEFAULT '',
  `redirectedUrl` varchar(1024) DEFAULT NULL,
  `title` varchar(255) DEFAULT NULL,
  `charSet` varchar(50) DEFAULT NULL,
  `tags` varchar(255) DEFAULT NULL,
  `depth` smallint(3) NOT NULL DEFAULT '0',
  `content` mediumblob,
  `lastVisitTime` datetime DEFAULT NULL,
  `lastFinishTime` datetime DEFAULT NULL,
  `type` smallint(6) DEFAULT NULL,
  `websiteId` int(11) NOT NULL DEFAULT '0',
  `md5` char(32) NOT NULL,
  PRIMARY KEY (`id`),
  UNIQUE KEY `md5` (`md5`),
  KEY `url` (`url`(200)),
  KEY `redirectedUrl` (`redirectedUrl`(200)),
  KEY `type` (`type`),
  KEY `siteId` (`websiteId`),
  KEY `depth` (`depth`)
) ENGINE=InnoDB  DEFAULT CHARSET=utf8;

#
# Source for table website
#

CREATE TABLE `website` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `siteName` varchar(255) DEFAULT NULL,
  `tags` varchar(1024) DEFAULT NULL,
  `desc` text,
  `indexPageId` int(11) DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;



CREATE TABLE `pic` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `url` varchar(1024) NOT NULL DEFAULT '',
  `data` MediumBlob,
  `lastModified` datetime DEFAULT NULL,
  PRIMARY KEY (`id`),
  UNIQUE KEY `url` (`url`(200))
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

CREATE TABLE `todo_pic` (
  `url` varchar(1024) NOT NULL DEFAULT '',
  `failcount` int NOT NULL DEFAULT 0,
  UNIQUE KEY `url` (`url`(200))
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

CREATE TABLE `crawl_log` (
  `id` bigint(11) NOT NULL AUTO_INCREMENT,
  `taskId` varchar(50) NOT NULL,
  `type` varchar(50) NOT NULL,
  `msg` text,
  `logTime` datetime,
  `level` tinyint,
  `content` mediumtext,
  PRIMARY KEY (`id`),
  KEY `taskId` (`taskId`),
  KEY `type` (`type`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;


CREATE TABLE `cmt_url` (
  `md5` char(32) NOT NULL,
  `url` varchar(1024) NOT NULL DEFAULT '',
  `status` tinyint(4) NOT NULL DEFAULT 0,
  `total_count` int(11) NOT NULL DEFAULT 0,
  `good_count` int(11) NOT NULL DEFAULT 0,
  `bad_count` int(11) NOT NULL DEFAULT 0,
  `neutral_count` int(11) NOT NULL DEFAULT 0,
  `tags` varchar(1024) DEFAULT NULL,
  `last_visit_time` datetime DEFAULT NULL,
  `last_finish_time` datetime DEFAULT NULL,
  `last_publish_time` datetime DEFAULT NULL,
  PRIMARY KEY (`md5`),
  KEY `status` (`status`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
	
CREATE TABLE `cmt_content` ( 
    `md5` char(32) NOT NULL,
	`user` varchar(255) NOT NULL, 
	`pub_time` bigint NOT NULL,
    `content` mediumtext,
	PRIMARY KEY (`md5`,`user`,`pub_time`), 
	KEY `md5` (`md5`)
	) ENGINE=InnoDB DEFAULT CHARSET=utf8;

CREATE TABLE `component_status` ( 
    `host_name_port` varchar(255) NOT NULL,
	`status` varchar(255) NOT NULL, 
	`last_update` datetime DEFAULT NULL,
	 PRIMARY KEY (`host_name_port`)
	) ENGINE=InnoDB DEFAULT CHARSET=utf8;
	
CREATE TABLE `review_status` ( 
    `page_id` int NOT NULL,
	`page_url` varchar(1024) NOT NULL, 
	`add_time` datetime NOT NULL,
    `lastUpdate` datetime,
    `lastestReviewTime` varchar(50),
    `lastAdded` int,
    `total_review` int default 0,
    `update_interval_sec` int,
    `crawling_status` int default 0,
	PRIMARY KEY (`page_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;	

CREATE TABLE `review_content` ( 
    `rv_id` varchar(255) NOT NULL,
    `page_id` int NOT NULL,
    `date` varchar(50),
    `content` mediumtext,
	PRIMARY KEY (`rv_id`),
	KEY `pid` (`page_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;		

insert into website(siteName,tags) values('凤凰网佛教频道','fojiao');
ALTER TABLE webpage AUTO_INCREMENT = 1;
insert into webpage(url,websiteId,md5) values('http://fo.ifeng.com',1,md5('http://fo.ifeng.com'));

update website set indexPageId=1 where id=1;

insert into website(siteName,tags) values('凤凰网佛教频道','fojiao');
ALTER TABLE webpage AUTO_INCREMENT = 1;
insert into webpage(url,websiteId,md5) values('http://fo.ifeng.com',1,md5('http://fo.ifeng.com'));

cd hornetq-2.4.0.Final/config/stand-alone/non-clustered

修改hornetq-beans.xml如下所示
      <property name="port">${jnp.port:1099}</property>
      <property name="bindAddress">${jnp.host:内网ip}</property>
      <property name="rmiPort">${jnp.rmiPort:1098}</property>
      <property name="rmiBindAddress">${jnp.host:内网ip}</property>

修改hornetq-configuration.xml，所以出现host和port的地方都需要修改，这里我只列举了一个！！
      <connector name="netty-throughput">
         <factory-class>org.hornetq.core.remoting.impl.netty.NettyConnectorFactory</factory-class>
         <param key="host"  value="${hornetq.remoting.netty.host:内网ip}"/>
         <param key="port"  value="${hornetq.remoting.netty.batch.port:5455}"/>
         <param key="batch-delay" value="50"/>
      </connector>
  
 需要修改<address-full-policy>PAGE</address-full-policy>
   <address-settings>
      <!--default for catch all-->
      <address-setting match="#">
         <dead-letter-address>jms.queue.DLQ</dead-letter-address>
         <expiry-address>jms.queue.ExpiryQueue</expiry-address>
         <redelivery-delay>0</redelivery-delay>
         <max-size-bytes>104857600</max-size-bytes>
         <message-counter-history-day-limit>10</message-counter-history-day-limit>
         <address-full-policy>PAGE</address-full-policy>
      </address-setting>
   </address-settings>

在hornetq-jms.xml增加：
   <queue name="ifeng" >
      <entry name="queues/ifeng"/>
   </queue>

#export JVM_ARGS="$CLUSTER_PROPS -XX:+UseParallelGC -XX:+AggressiveOpts -XX:+UseFastAccessorMethods -Xms512M -Xmx1024M -Dhornetq.config.dir=$RESOLVED_CONFIG_DIR -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Dlogging.configuration=file://$RESOLVED_CONFIG_DIR/logging.properties -Djava.library.path=./lib/linux-i686:./lib/linux-x86_64"
export JVM_ARGS="$CLUSTER_PROPS -XX:+UseParallelGC -XX:+AggressiveOpts -XX:+UseFastAccessorMethods -Xms512M -Xmx2048M -Dhornetq.config.dir=$RESOLVED_CONFIG_DIR -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Dlogging.configuration=file://$RESOLVED_CONFIG_DIR/logging.properties -Djava.library.path=./lib/linux-i686:./lib/linux-x86_64  -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=3333 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=内网ip"

	private Extractor[] extractors = new Extractor[] {
			new Level0Extractor(),
			new ListPageExtractor(),
			new DetailPageExtractor(),
	};

	@Override
	public Extractor getExtractor(String url, String redirectedUrl, int depth) {
		if (depth == 0)
			return extractors[0];
		else if(depth == 1)
			return extractors[1];
		else if(depth == 2)
			return extractors[2];
		return null;
	}
	

public interface Extractor {
	public void extractProps(WebPage webPage, NekoHtmlParser parser, HttpClientFetcher fetcher, String content,
			Archiver archiver, String taskId);

	public List<Block> extractBlock(WebPage webPage, NekoHtmlParser parser, HttpClientFetcher fetcher, String content,
			Archiver archiver, String taskId);

	public void extractBasicInfo(WebPage webPage, String content, Archiver archiver, String taskId);

	public boolean needUpdate(WebPage webPage);

	public boolean needSaveHtml(WebPage webPage);

	public boolean needAddChildren2FrontierIfNotUpdate(WebPage webPage);

}

	@Override
	public boolean needUpdate(WebPage webPage) {
		return super.needUpdate(webPage.getLastVisitTime(), 15, 0, 0, 0);
	}

	protected boolean needUpdate(Date lastUpdate, int days, int hours, int minutes, int seconds) {
		if (lastUpdate == null)
			return true;
		long interval = 1000L * (seconds + 60 * minutes + 3600 * hours + 24 * 3600 * days);
		if (interval <= 0) {
			interval = Long.MAX_VALUE;
		}
		return System.currentTimeMillis() - lastUpdate.getTime() > interval;
	}

	@Override
	public boolean needSaveHtml(WebPage webPage) {
		return true;
	};

	@Override
	public boolean needAddChildren2FrontierIfNotUpdate(WebPage webPage) {
		return true;
	}
	
	@Override
	public void extractBasicInfo(WebPage webPage, String content, Archiver archiver, String taskId) {
		...

	}

	protected Block addChild(List<String> urls, List<String> anchors, int depth,
			List<Map<String, String>> attrsFromParent, boolean removeEmptyAnchor, boolean removeAnchorInUrl,
			boolean dedup, boolean encodeChinese) {
			
	}

protected String normUrl(String url) {
		return url;
	}

protected static String rewriteUrl(String oldUrl, String key, String value)

		String oldUrl = "http://shouji.baidu.com/software/list?page=1&cid=506";
		String newUrl = rewriteUrl(oldUrl, "page", "5");

 
 public class Level0Extractor extends IfengBasicInfoExtractor {
	protected static Logger logger = Logger.getLogger(Level0Extractor.class);

	@Override
	public void extractProps(WebPage webPage, NekoHtmlParser parser, HttpClientFetcher fetcher, String content,
			Archiver archiver, String taskId) {

	}

	@Override
	public List<Block> extractBlock(WebPage webPage, NekoHtmlParser parser, HttpClientFetcher fetcher, String content,
			Archiver archiver, String taskId) {
		List<Block> blocks = new ArrayList<>(1);
		List<String> urls = new ArrayList<>();
		List<String> anchors = new ArrayList<>();

		NodeList aList = parser.selectNodes("//DIV[@id='col_wbf']//LI/A");
		for (int i = 0; i < aList.getLength(); i++) {
			Node a = aList.item(i);
			String anchor = a.getTextContent().trim();
			String href = parser.getNodeText("./@href", a);
			String url = UrlUtils.getAbsoluteUrl(webPage.getUrl(), href);
			if (url.equals("http://fo.ifeng.com/"))
				continue;
			urls.add(url);
			anchors.add(anchor);
		}

		blocks.add(super.addChild(urls, anchors, webPage.getDepth() + 1));
		return blocks;
	}

	@Override
	public boolean needUpdate(WebPage webPage) {
		return true;
	}

	@Override
	public boolean needAddChildren2FrontierIfNotUpdate(WebPage webPage) {
		return true;
	}

	public static void main(String[] args) {
		String[] urls = new String[] { "http://fo.ifeng.com", };
		Level0Extractor ext = new Level0Extractor();
		for (String url : urls) {
			ext.testUrl(url);
		}
	}
	

		NodeList aList = parser.selectNodes("//DIV[@id='col_wbf']//LI/A");
		for (int i = 0; i < aList.getLength(); i++) {
			Node a = aList.item(i);
			String anchor = a.getTextContent().trim();
			String href = parser.getNodeText("./@href", a);
			String url = UrlUtils.getAbsoluteUrl(webPage.getUrl(), href);
			if (url.equals("http://fo.ifeng.com/"))
				continue;
			urls.add(url);
			anchors.add(anchor);
		}
 

	public List<ListPageItem> extractItems(String url, NekoHtmlParser parser, Archiver archiver, String taskId,
			String content) {
		List<ListPageItem> items = new ArrayList<>(10);
		NodeList divs = parser.selectNodes("//DIV[@class='col_L']/DIV[1]/DIV");
		if (divs == null) {
			archiver.saveLog(taskId, "ListPageExtractor.extractItems", "divs==null", new Date(),
					GlobalConstants.LOG_LEVEL_WARN, url + "##" + content);
		} else {
			for (int i = 0; i < divs.getLength(); i++) {
				Node div = divs.item(i);
				String href = parser.getNodeText("./H2/A/@href", div);
				String itemUrl = UrlUtils.getAbsoluteUrl(url, href);
				String title = parser.getNodeText("./H2/A", div);
				String pubTime = parser.getNodeText("./DIV[contains(@class,'box_txt')]/SPAN", div);
				ListPageItem listPageItem = new ListPageItem(title, itemUrl, pubTime);
				items.add(listPageItem);
			}
		}
		return items;
	}
	

	public static String pubTimeBound="2015-01-01";
	
		
		String timeBound=pubTimeBound;
		if(webPage.getLastFinishTime()!=null){
			timeBound = DateTimeTools.formatDate(webPage.getLastFinishTime());
			logger.info("useLastFinishTime: "+timeBound+" for: "+webPage.getUrl());
		}

		//http://fo.ifeng.com/listpage/119/1/list.shtml
		String url=webPage.getUrl();
		String[] urlLeftRightPart=this.parseListUrl(url);
		if(urlLeftRightPart==null){
			archiver.saveLog(taskId, "ListPageExtractor.extractBlock", "urlLeftRightPart", new Date(),
					GlobalConstants.LOG_LEVEL_WARN, url);
			return blocks;
		}

		//process firstPage
		List<ListPageItem> allItems=this.extractItems(url, parser, archiver, taskId, content);		
		

		for(int i=2;i<=maxPage;i+=maxCrawlThread){
			int startPage=i;
			int endPage=Math.min(maxPage, i+maxCrawlThread);
			List<String> batchUrls=new ArrayList<>(maxCrawlThread);
			for(int pg=startPage;pg<endPage;pg++){
				String pageUrl=urlLeftRightPart[0]+pg+urlLeftRightPart[1];
				batchUrls.add(pageUrl);
			}
			List<String[]> batchResults=BatchCrawler.crawler(batchUrls, maxCrawlThread, fetcher, cpi);
			boolean needContinue=true;
			for(String[] pair:batchResults){
				if(pair[1]==null){
					archiver.saveLog(taskId, "ListPageExtractor.extractBlock", "batchResults", new Date(),
							GlobalConstants.LOG_LEVEL_WARN, pair[0]);
					continue;
				}
				try {
					parser.load(pair[1], "UTF8");					
				} catch (Exception e) {
					archiver.saveLog(taskId, "ListPageExtractor.extractBlock", "parser.load", new Date(),
							GlobalConstants.LOG_LEVEL_WARN, pair[0]);
					continue;
				}
				
				List<ListPageItem> items = this.extractItems(pair[0], parser, archiver, taskId, pair[1]);
				allItems.addAll(items);
				for(ListPageItem item:items){
					if(needContinue && item.getPubTime().compareTo(timeBound)<0){
						logger.info("skip older: "+item);
						needContinue=false;
					}
				}
			}
			
			if(!needContinue) break;
		}
		

			boolean needContinue=true;
			for(String[] pair:batchResults){
				if(pair[1]==null){
					archiver.saveLog(taskId, "ListPageExtractor.extractBlock", "batchResults", new Date(),
							GlobalConstants.LOG_LEVEL_WARN, pair[0]);
					continue;
				}
				try {
					parser.load(pair[1], "UTF8");					
				} catch (Exception e) {
					archiver.saveLog(taskId, "ListPageExtractor.extractBlock", "parser.load", new Date(),
							GlobalConstants.LOG_LEVEL_WARN, pair[0]);
					continue;
				}
				
				List<ListPageItem> items = this.extractItems(pair[0], parser, archiver, taskId, pair[1]);
				allItems.addAll(items);
				for(ListPageItem item:items){
					if(needContinue && item.getPubTime().compareTo(timeBound)<0){
						logger.info("skip older: "+item);
						needContinue=false;
					}
				}
			}

List<ListPageItem> items = this.extractItems(pair[0], parser, archiver, taskId, pair[1]);
allItems.addAll(items);	

public static int maxPage=500;
		for(int i=2;i<=maxPage;i+=maxCrawlThread){
			...
		}

		Collections.sort(allItems, new Comparator<ListPageItem>(){
			@Override
			public int compare(ListPageItem o1, ListPageItem o2) {
				return o2.getPubTime().compareTo(o1.getPubTime());
			}	
		});
		for(ListPageItem item:allItems){
			urls.add(item.getUrl());
			anchors.add(item.getTitle());
		}
		//update finish time
		if(allItems.size()>0){
			String putTime=allItems.iterator().next().getPubTime();
			Date d=DateTimeTools.parseDate(putTime);
			if(d==null){
				archiver.saveLog(taskId, "ListPageExtractor.extractBlock", "parseDate", new Date(),
						GlobalConstants.LOG_LEVEL_WARN, url);
			}else{
				Calendar c = Calendar.getInstance();
				c.setTime(d);
				//minus one hour to avoid same time in ifeng(it's yyyy-MM-dd HH:mm)
				c.add(Calendar.HOUR_OF_DAY, -1);
				d=c.getTime();
				//webPage.setLastFinishTime(d);
				archiver.updateFinishTime(webPage, d);
				logger.info("setLastFinish: "+webPage.getUrl()+"\t"+DateTimeTools.formatDate(d));
			}
		}

		blocks.add(super.addChild(urls, anchors, webPage.getDepth() + 1));
		return blocks;

	@Override
	public void extractProps(WebPage webPage, NekoHtmlParser parser, HttpClientFetcher fetcher, String content,
			Archiver archiver, String taskId) {
		String title=parser.getNodeText("//H1");
		Map<String, String> attrs = webPage.getAttrs();
		if(attrs==null){
			attrs = new HashMap<>();
			webPage.setAttrs(attrs);
		}
		attrs.put("#title#", title);
	}

	@Override
	public List<Block> extractBlock(WebPage webPage, NekoHtmlParser parser, HttpClientFetcher fetcher, String content,
			Archiver archiver, String taskId) {
		return null;
	}

public class CrawlerDriver {

	public static void main(String[] args) throws Exception {
		BasicCrawlerWithFileFrontier.main(args);
	}

}

public class SchedulerDriver {
	public static void main(String[] args) throws Exception {
		PoolManager.StartPool("conf", "baidumusic");
		FileUnfinishedScheduler.main(args);
	}
}

把ai-dev改成你的mysql服务器的内网ip，用户名密码也要修改
MYSQL_URL=jdbc:mysql://ai-dev:3306/${db}?useUnicode=true&characterEncoding=utf-8
MYSQL_USER=ifeng
MYSQL_PASS=12345

java -Dlog4j.configuration=log4j-init.properties -cp ".:../ifeng_crawler-1.0.1-jar-with-dependencies.jar" com.antbrains.ifengcrawler.scheduler.IfengInit  "jnp://ai-dev:1099" ai-dev:3333 ifeng 15m 5m &

	CREATE TABLE `component_status` ( 
    `host_name_port` varchar(255) NOT NULL,
	`status` varchar(255) NOT NULL, 
	`last_update` datetime DEFAULT NULL,
	 PRIMARY KEY (`host_name_port`)
	) ENGINE=InnoDB DEFAULT CHARSET=utf8;

~~需要把ai-dev改成你的服务器的内网ip！！！~~
~~  java -Dlog4j.configuration=log4j-init.properties -cp ".:./ifeng_crawler-1.0-jar-with-dependencies.jar" com.antbrains.ifengcrawler.scheduler.IfengScheduler  "jnp://ai-dev:1099" ai-dev:3333 ifeng~~

同样需要修改ai-dev！
--workerNum=10 指定爬虫的抓取线程个数
com.antbrains.ifengcrawler.extractor.IfengExtractor 就是我们实现的UrlPatternExtractor接口的类的全限定路径名，告诉我们的框架应该怎么处理url

java  -Dlog4j.configuration=log4j-crawl.properties -cp ".:./ifeng_crawler-1.0-jar-with-dependencies.jar" com.antbrains.ifengcrawler.scheduler.CrawlerDriver --isUpdate true --workerNum 10 "jnp://ai-dev:1099" ai-dev:3333 ifeng com.antbrains.ifengcrawler.extractor.IfengExtractor queuedata &

把ai-dev改成你的hornetq的名称，把ifeng改成你的队列名
java -cp ".:./mqtool-1.0-jar-with-dependencies.jar" com.antbrains.mqtool.test.QueueSize "jnp://ai-dev:1099" ai-dev:3333 ifeng
 
clear_queue.sh 清空队列

java -cp ".:./mqtool-1.0-jar-with-dependencies.jar" com.antbrains.mqtool.test.EmptyQueue "jnp://ai-dev:1099" ai-dev:3333 ifeng

cd knowledge-distillation/scui
修改src/main/resources/cfg.properties里的mysql的服务器host，帐号和密码

mvn package
wget 'http://central.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.3.13.v20161014/jetty-runner-9.3.13.v20161014.jar'

然后运行 java -jar jetty-runner-9.3.13.v20161014.jar target/scui.war