Web Crawler Crawl4j

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Set;

public class Mycrawler extends WebCrawler {
    static double totalFilesCount = 0.0;
    static  File file = new File("Links.txt");
    public  static  void writelinkstoFile(String url){
        try {
            BufferedWriter bf  = new BufferedWriter(new FileWriter(file,true));
            bf.write(url+"\n");
            bf.flush();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * This method receives two parameters. The first parameter is the page
     * in which we have discovered this new url and the second parameter is
     * the new url. You should implement this function to specify whether
     * the given url should be crawled or not (based on your crawling logic).
     * In this example, we are instructing the crawler to ignore urls that
     * have css, js, git, ... extensions and to only accept urls that start
     * with "http://www.ics.uci.edu/". In this case, we didn't need the
     * referringPage parameter to make the decision.
     */
    public boolean checkifIsDIR(String href){
        if (href.endsWith("/")){
            return true;
        }else
        return false;

    }
    @Override
    public boolean shouldVisit(Page referringPage, WebURL url) {
        String orghref =url.getURL();
        String href = url.getURL().toLowerCase();

        boolean iswithinDomain  =href.startsWith(Controller.TargetSite.toLowerCase());
        if (iswithinDomain&&checkifIsDIR(href)){
          //  System.out.println(href);
            return true;
        }else if (iswithinDomain){
            {
                if(href.equalsIgnoreCase("http://hacking.technology/Hacked%20Team/Exploit_Delivery_Network_android.tar.gz")){
                    return false;
                }
                if(href.equalsIgnoreCase("http://hacking.technology/Hacked%20Team/Exploit_Delivery_Network_windows.tar.gz")){
                    return false;
                }
                if(href.equalsIgnoreCase("http://hacking.technology/Hacked%20Team/support.hackingteam.com.tar.gz")){
                    return false;
                }

            }
            writelinkstoFile(orghref);
            totalFilesCount++;
        }
        return false;

    }

    /**
     * This function is called when a page is fetched and ready
     * to be processed by your program.
     */
    @Override
    public void visit(Page page) {
        String url = page.getWebURL().getURL();
        System.out.println("visited URL: " + url);

        if (page.getParseData() instanceof HtmlParseData) {
            HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
            String text = htmlParseData.getText();
            String html = htmlParseData.getHtml();
            Set<WebURL> links = htmlParseData.getOutgoingUrls();

            System.out.println("Text length: " + text.length());
            System.out.println("Html length: " + html.length());
            System.out.println("Number of outgoing links: " + links.size());
        }
    }
}

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


import java.util.Properties;

/**
 * Created by jasonzhuo on 2015/7/11.
 */
public class Controller {
    public  static String TargetSite= "http://hacking.technology/Hacked%20Team/";
    private static final Logger logger = LoggerFactory.getLogger(Controller.class);

    static int numberOfCrawlers = 7;

    public static void main(String[] args) {

        CrawlConfig crawlConfig = new CrawlConfig();
        crawlConfig.setCrawlStorageFolder("I:\\HTools");

        //proxy setup
       //crawlConfig.setProxyHost("127.0.0.1");
      //  crawlConfig.setProxyPort(1080);
        Properties prop = System.getProperties();
        prop.setProperty("socksProxyHost", "127.0.0.1");
        prop.setProperty("socksProxyPort", "1080");
        crawlConfig.setMaxDownloadSize(Integer.MAX_VALUE);
        crawlConfig.setMaxOutgoingLinksToFollow(Integer.MAX_VALUE);
        crawlConfig.setMaxDepthOfCrawling(32767);
        crawlConfig.setMaxPagesToFetch(Integer.MAX_VALUE);
        System.out.println(crawlConfig.toString());
        PageFetcher pageFetcher = new PageFetcher(crawlConfig);
        RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
        RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
        try {
            CrawlController controller = new CrawlController(crawlConfig, pageFetcher, robotstxtServer);
            //controller.addSeed("http://www.ics.uci.edu/~welling/");
            controller.addSeed(TargetSite);
            // controller.addSeed("https://www.google.com.hk/");
            controller.start(Mycrawler.class, numberOfCrawlers);
            System.out.println("Recorded total number of files :"+Mycrawler.totalFilesCount);
        } catch (Exception e) {
            e.printStackTrace();
        }


    }


}