从头学习爬虫（二十五）重构篇----WebMagic框架分析之细节

这个框架有很多细节等着我们去发现,本文主要介绍下。

线程池设计

对于小白来说可以好好学习下

package us.codecraft.webmagic.thread;

import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;

/**
 * Thread pool for workers.<br><br>
 * Use {@link java.util.concurrent.ExecutorService} as inner implement. <br><br>
 * New feature: <br><br>
 * 1. Block when thread pool is full to avoid poll many urls without process. <br><br>
 * 2. Count of thread alive for monitor.
 *
 * @author [email protected]
 * @since 0.5.0
 */
public class CountableThreadPool {

    private int threadNum;

    private AtomicInteger threadAlive = new AtomicInteger();

    private ReentrantLock reentrantLock = new ReentrantLock();

    private Condition condition = reentrantLock.newCondition();

    public CountableThreadPool(int threadNum) {
        this.threadNum = threadNum;
        this.executorService = Executors.newFixedThreadPool(threadNum);
    }

    public CountableThreadPool(int threadNum, ExecutorService executorService) {
        this.threadNum = threadNum;
        this.executorService = executorService;
    }

    public void setExecutorService(ExecutorService executorService) {
        this.executorService = executorService;
    }

    public int getThreadAlive() {
        return threadAlive.get();
    }

    public int getThreadNum() {
        return threadNum;
    }

    private ExecutorService executorService;

    public void execute(final Runnable runnable) {


        if (threadAlive.get() >= threadNum) {
            try {
                reentrantLock.lock();
                while (threadAlive.get() >= threadNum) {
                    try {
                        condition.await();
                    } catch (InterruptedException e) {
                    }
                }
            } finally {
                reentrantLock.unlock();
            }
        }
        threadAlive.incrementAndGet();
        executorService.execute(new Runnable() {
            @Override
            public void run() {
                try {
                    runnable.run();
                } finally {
                    try {
                        reentrantLock.lock();
                        threadAlive.decrementAndGet();
                        condition.signal();
                    } finally {
                        reentrantLock.unlock();
                    }
                }
            }
        });
    }

    public boolean isShutdown() {
        return executorService.isShutdown();
    }

    public void shutdown() {
        executorService.shutdown();
    }


}

代理池设计

这块我觉得需要很大的改造

package us.codecraft.webmagic.proxy;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Task;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * A simple ProxyProvider. Provide proxy as round-robin without heartbeat and error check. It can be used when all proxies are stable.
 * @author [email protected]
 *         Date: 17/4/16
 *         Time: 10:18
 * @since 0.7.0
 */
public class SimpleProxyProvider implements ProxyProvider {

    private final List<Proxy> proxies;

    private final AtomicInteger pointer;

    public SimpleProxyProvider(List<Proxy> proxies) {
        this(proxies, new AtomicInteger(-1));
    }

    private SimpleProxyProvider(List<Proxy> proxies, AtomicInteger pointer) {
        this.proxies = proxies;
        this.pointer = pointer;
    }

    public static SimpleProxyProvider from(Proxy... proxies) {
        List<Proxy> proxiesTemp = new ArrayList<Proxy>(proxies.length);
        for (Proxy proxy : proxies) {
            proxiesTemp.add(proxy);
        }
        return new SimpleProxyProvider(Collections.unmodifiableList(proxiesTemp));
    }

    @Override
    public void returnProxy(Proxy proxy, Page page, Task task) {
        //Donothing
    }

    @Override
    public Proxy getProxy(Task task) {
        return proxies.get(incrForLoop());
    }

    private int incrForLoop() {
        int p = pointer.incrementAndGet();
        int size = proxies.size();
        if (p < size) {
            return p;
        }
        while (!pointer.compareAndSet(p, p % size)) {
            p = pointer.get();
        }
        return p % size;
    }
}

选择器

这块xpath,部分语法不支持，但是你还可以发现还有个xpath2

package us.codecraft.webmagic.selector;

import net.sf.saxon.lib.NamespaceConstant;
import net.sf.saxon.xpath.XPathEvaluator;
import org.apache.log4j.Logger;
import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.DomSerializer;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import javax.xml.namespace.NamespaceContext;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

/**
 * 支持xpath2.0的选择器。包装了HtmlCleaner和Saxon HE。<br>
 *
 * @author [email protected] <br>
 *         Date: 13-4-21
 *         Time: 上午9:39
 */
public class Xpath2Selector implements Selector {

    private String xpathStr;

    private XPathExpression xPathExpression;

    private Logger logger = Logger.getLogger(getClass());

    public Xpath2Selector(String xpathStr) {
        this.xpathStr = xpathStr;
        try {
            init();
        } catch (XPathExpressionException e) {
            throw new IllegalArgumentException("XPath error!", e);
        }
    }

    enum XPath2NamespaceContext implements NamespaceContext {

        INSTANCE;

        private final Map<String, String> prefix2NamespaceMap = new ConcurrentHashMap<String, String>();

        private final Map<String, List<String>> namespace2PrefixMap = new ConcurrentHashMap<String, List<String>>();

        private void put(String prefix, String namespaceURI) {
            prefix2NamespaceMap.put(prefix, namespaceURI);
            List<String> prefixes = namespace2PrefixMap.get(namespaceURI);
            if (prefixes == null) {
                prefixes = new ArrayList<String>();
                namespace2PrefixMap.put(namespaceURI, prefixes);
            }
            prefixes.add(prefix);
        }

        private XPath2NamespaceContext() {
            put("fn", NamespaceConstant.FN);
            put("xslt", NamespaceConstant.XSLT);
        }

        @Override
        public String getNamespaceURI(String prefix) {
            return prefix2NamespaceMap.get(prefix);
        }

        @Override
        public String getPrefix(String namespaceURI) {
            List<String> prefixes = namespace2PrefixMap.get(namespaceURI);
            if (prefixes == null || prefixes.size() < 1) {
                return null;
            }
            return prefixes.get(0);
        }

        @Override
        public Iterator getPrefixes(String namespaceURI) {
            List<String> prefixes = namespace2PrefixMap.get(namespaceURI);
            if (prefixes == null || prefixes.size() < 1) {
                return null;
            }
            return prefixes.iterator();
        }
    }

    private void init() throws XPathExpressionException {
        XPathEvaluator xPathEvaluator = new XPathEvaluator();
        xPathEvaluator.setNamespaceContext(XPath2NamespaceContext.INSTANCE);
        xPathExpression = xPathEvaluator.compile(xpathStr);
    }

    @Override
    public String select(String text) {
        try {
            HtmlCleaner htmlCleaner = new HtmlCleaner();
            TagNode tagNode = htmlCleaner.clean(text);
            Document document = new DomSerializer(new CleanerProperties()).createDOM(tagNode);
            Object result;
            try {
                result = xPathExpression.evaluate(document, XPathConstants.NODESET);
            } catch (XPathExpressionException e) {
                result = xPathExpression.evaluate(document, XPathConstants.STRING);
            }
            if (result instanceof NodeList) {
                NodeList nodeList = (NodeList) result;
                if (nodeList.getLength() == 0) {
                    return null;
                }
                Node item = nodeList.item(0);
                if (item.getNodeType() == Node.ATTRIBUTE_NODE || item.getNodeType() == Node.TEXT_NODE) {
                    return item.getTextContent();
                } else {
                    StreamResult xmlOutput = new StreamResult(new StringWriter());
                    Transformer transformer = TransformerFactory.newInstance().newTransformer();
                    transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
                    transformer.transform(new DOMSource(item), xmlOutput);
                    return xmlOutput.getWriter().toString();
                }
            }
            return result.toString();
        } catch (Exception e) {
            logger.error("select text error! " + xpathStr, e);
        }
        return null;
    }

    @Override
    public List<String> selectList(String text) {
        List<String> results = new ArrayList<String>();
        try {
            HtmlCleaner htmlCleaner = new HtmlCleaner();
            TagNode tagNode = htmlCleaner.clean(text);
            Document document = new DomSerializer(new CleanerProperties()).createDOM(tagNode);
            Object result;
            try {
                result = xPathExpression.evaluate(document, XPathConstants.NODESET);
            } catch (XPathExpressionException e) {
                result = xPathExpression.evaluate(document, XPathConstants.STRING);
            }
            if (result instanceof NodeList) {
                NodeList nodeList = (NodeList) result;
                Transformer transformer = TransformerFactory.newInstance().newTransformer();
                StreamResult xmlOutput = new StreamResult();
                transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
                for (int i = 0; i < nodeList.getLength(); i++) {
                    Node item = nodeList.item(i);
                    if (item.getNodeType() == Node.ATTRIBUTE_NODE || item.getNodeType() == Node.TEXT_NODE) {
                        results.add(item.getTextContent());
                    } else {
                        xmlOutput.setWriter(new StringWriter());
                        transformer.transform(new DOMSource(item), xmlOutput);
                        results.add(xmlOutput.getWriter().toString());
                    }
                }
            } else {
                results.add(result.toString());
            }
        } catch (Exception e) {
            logger.error("select text error! " + xpathStr, e);
        }
        return results;
    }
}

分布式

这块没有成熟demo，很可惜

webmagic-avalon

这个项目也没有完成，但是有别的项目代替，爬虫管理是重点。

重点

这一系列文章，剖析了整个框架，以及爬虫如何编写。

我们可以从实现方式，制定3种方案

1 全模拟 全程使用驱动去完成爬取过程，优点：操作简单 缺点：速度慢，即便用了headless

2 半模拟 在登入获取cookie等值，或者中途去执行js，优点：中等难度，速度较快，易于迅速解决问题 缺点：速度较慢

3 无模拟 仅凭代码实现各种参数生成，优点：速度很快，缺点：真的很烦，解析Js最烦

欢迎加群313557283（刚创建），小白互相学习~，如果觉得这系列文章很好，想要打赏，可以加群发红包给管理员

前几篇打好基础，后面我们有机会用python实战下