金融类数据测试与监控实践 - 51Testing软件测试论坛

QuartzManager.addJob("btctrade",
BtcTradeJob.class.getName(), "*/5 * 7-23 * * ?"); //7-23点获取数据，定时5秒刷新接口
QuartzManager.addJob("次新股",
NewStockPlateJob.class.getName(), "*/5 * 9-15 ? * MON-FRI");//9点15开始定时5秒刷新接口
定制获取数据的job，以比特币为例子
package com.mj.job;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import us.codecraft.webmagic.Spider;
import com.mj.processor.BtcProcessor;
import com.mj.processor.BtcTradeProcessor;
public class BtcTradeJob implements Job {
// BTC：http://api.btctrade.com/api/ticker?coin=btc
// ETH：http://api.btctrade.com/api/ticker?coin=eth
// LTC：http://api.btctrade.com/api/ticker?coin=ltc
// DOGE：http://api.btctrade.com/api/ticker?coin=doge
// YBC：http://api.btctrade.com/api/ticker?coin=ybc
public void execute(JobExecutionContext arg0) throws JobExecutionException {
Spider.create(new BtcTradeProcessor())
.addUrl("http://api.btctrade.com/api/ticker?coin=btc",
"http://api.btctrade.com/api/ticker?coin=eth",
"http://api.btctrade.com/api/ticker?coin=ltc",
"http://api.btctrade.com/api/ticker?coin=doge",
"http://api.btctrade.com/api/ticker?coin=ybc")
.thread(2)
.run();
}
}
通过 Processor 进行对比
package com.mj.processor;
import java.util.Date;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.mj.activity.BtcServer;
import com.mj.common.cache.CacheManager;
import com.mj.model.BtcModel;
import com.mj.model.NotifyModel;
import com.mj.util.date.VeDate;
import com.mj.util.mail.MailUtil;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
public class BtcTradeProcessor implements PageProcessor {
private static final Logger logger = LoggerFactory
.getLogger(BtcTradeProcessor.class);
// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
private Site site = Site.me().setRetryTimes(5).setSleepTime(1000);
// process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
public void process(Page page) {
// TODO Auto-generated method stub
// page.setSkip(true);
if (page != null) {
logger.debug(page.getJson().toString());
// MailUtil.mail("数据", page.getJson().toString());
BtcModel model = JSON.parseObject(page.getJson().toString(),
BtcModel.class);
if (model != null) {
model.type = page.getRequest().getUrl().replace("http://api.btctrade.com/api/ticker?coin=", "btctrade_");
logger.debug(model.toString());
if (CacheManager.getInstance().getCache(model.type) == null) {
CacheManager.getInstance().putCache(model.type,
"20170101010101");
}
//这里忽略了公司的对比代码
....
if (不通过就发送邮件) {
CacheManager.getInstance().putCache(model.type,
nowStr);
NotifyModel notifyModel = new NotifyModel();
notifyModel.head = model.type + "最低价";
notifyModel.body = " 最新 " + model.last + " 最低 "
+ model.low + " 最高 " + model.high +“在“+nowStr+”出现数据不一致”;
MailUtil.mail(notifyModel.head, notifyModel.body);
}
}
}
}
public Site getSite() {
return site;
}
}

复制代码