java - Getting many memory errors when try to run it for few days in my web crawler -

i developing web crawler application. when run program getting these error messages below:

i've got these errors after running program more 3 hours. tried allocate memory changing eclipse.ini setting 2048 mb of ram answered in this topic still same errors after 3 hours or less. should run program more 2-3 days non-stopping analyse results.

can tell me missing here these error below ?

these classes:

seeds.txt

http://www.stanford.edu http://www.archive.org 

webcrawler.java

 package pkg.crawler;  import java.io.bufferedwriter; import java.io.file; import java.io.filewriter; import java.io.ioexception; import java.io.printwriter; import java.net.malformedurlexception; import java.net.sockettimeoutexception; import java.util.*; import java.util.concurrent.executorservice; import java.util.concurrent.priorityblockingqueue; import java.util.concurrent.timeunit;  import org.jsoup.httpstatusexception; import org.jsoup.unsupportedmimetypeexception; import org.joda.time.datetime;   public class webcrawler {  public static queue <linknodelight> queue = new priorityblockingqueue <> (); // priority queue public static final int n_threads = 5;                                 // amount of threads private static set<string> processed = new linkedhashset <> ();         // set of processed urls private printwriter out;                                                // output file private printwriter err;                                                // error file private static integer cntintra = new integer (0);                              // counters intra- links in queue private static integer cntinter = new integer (0);                              // counters inter- links in queue private static integer dub = new integer (0);                                   // amount of skipped urls  public static void main(string[] args) throws exception {     system.out.println("running web crawler: " + new date());      webcrawler webcrawler = new webcrawler();     webcrawler.createfiles();     try (scanner in = new scanner(new file ("seeds.txt"))) {         while (in.hasnext()) {             webcrawler.enque(new linknode (in.nextline().trim()));         }     } catch (ioexception e) {         e.printstacktrace();         return;     }     webcrawler.processqueue();     webcrawler.out.close();     webcrawler.err.close(); }  public void processqueue(){     /* run in threads */     runnable r = new runnable() {         @override          public void run() {             /* queue may empty process not finished, that's why need check if links being processed */             while (true) {                 linknode link = deque();                 if (link == null)                     continue;                 link.setstarttime(new datetime());                 boolean process = processlink(link);                 link.setendtime(new datetime());                 if (!process)                     continue;                 /* print data csv file */                 if (link.getstatus() != null && link.getstatus().equals(linknodestatus.ok)) {                     synchronized(out) {                         out.println(getoutputline(link));                         out.flush();                     }                 } else {                     synchronized(err) {                         err.println(getoutputline(link));                         err.flush();                     }                 }             }         }     };     /* run n_threads threads perform dequeue , process */     linkedlist <thread> threads = new linkedlist <> ();     (int = 0; < n_threads; i++) {         threads.add(new thread(r));         threads.getlast().start();     }     (thread thread : threads) {         try {             thread.join();         } catch (interruptedexception e) {             e.printstacktrace();         }     } }   /* returns true if link processed */ private boolean processlink(linknode inputlink) {     string url = geturlgeneralform(inputlink);     boolean process = true;     synchronized (processed) {         if (processed.contains(url)) {             process = false;             synchronized (dub) {dub++;}         } else             processed.add(url);     }     /* start processing if url have not been processed yet or not being processed */     if (process) {         system.out.println("processing url " + url);         list<linknodelight> outputlinks = parseandwieghtresults(inputlink);         (linknodelight outputlink : outputlinks) {             string geturlgeneralforumoutput = geturlgeneralform(outputlink);             /* add new link queue if has not been processed yet */             process = true;             synchronized (processed) {                 if (processed.contains(geturlgeneralforumoutput)) {                     process = false;                     synchronized (dub) {dub++;}                 }             }             if (process) {                 enque(outputlink);             }         }         return true;     }     return false; }  void enque(linknodelight link){     link.setenquetime(new datetime());     /* add method requires implicit priority */     synchronized (queue) {         if (link.interlinks)             synchronized (cntinter) {cntinter++;}         else             synchronized (cntintra) {cntintra++;}       //queue.add(link, 100 - (int)(link.getweight() * 100.f));         queue.add(link);     } }   /**  * picks element queue  * @return top element queue or null if queue empty  */ linknode deque(){     /* link must checked */     linknode link = null;     synchronized (queue) {         link = (linknode) queue.poll();         if (link != null) {             link.setdequetime(new datetime());             if (link.isinterlinks())                 synchronized (cntinter) {cntinter--;}             else                 synchronized (cntintra) {cntintra--;}         }     }     return link; }  private void createfiles() {     /* create output file */     try {         out = new printwriter(new bufferedwriter(new filewriter("crawledurls.csv", false)));         out.println(generateheaderfile());     } catch (ioexception e) {         system.err.println(e);     }     /* create error file */     try {         err = new printwriter(new bufferedwriter(new filewriter("crawledurlserror.csv", false)));         err.println(generateheaderfile());     } catch (ioexception e) {         system.err.println(e);     } } /**  * formats string can valid entry in csv file  * @param s  * @return  */ private static string format(string s) {     // replace " ""     string ret = s.replaceall("\"", "\"\"");     // put string quotes     return "\"" + ret + "\""; } /**  * creates line needs written in outputfile  * @param link  * @return  */ public static string getoutputline(linknode link){     stringbuilder builder = new stringbuilder();     builder.append(link.getparentlink()!=null ? format(link.getparentlink().geturl()) : "");     builder.append(",");     builder.append(link.getparentlink()!=null ? link.getparentlink().getipadress() : "");     builder.append(",");     builder.append(link.getparentlink()!=null ? link.getparentlink().linkprocessingduration() : "");     builder.append(",");     builder.append(format(link.geturl()));     builder.append(",");     builder.append(link.getdomain());     builder.append(",");     builder.append(link.isinterlinks());     builder.append(",");     builder.append(util.formatdate(link.getenquetime()));     builder.append(",");     builder.append(util.formatdate(link.getdequetime()));     builder.append(",");     builder.append(link.waitinginqueue());     builder.append(",");     builder.append(queue.size());     /* inter , intra links in queue */     builder.append(",");     builder.append(cntintra.tostring());     builder.append(",");     builder.append(cntinter.tostring());     builder.append(",");     builder.append(dub);     builder.append(",");     builder.append(new date ());     /* url size*/     builder.append(",");     builder.append(link.getsize());     /* html file     builder.append(",");     builder.append(link.getfilename());*/     /* add http error */     builder.append(",");     if (link.getparseexception() != null) {         if (link.getparseexception() instanceof httpstatusexception)             builder.append(((httpstatusexception) link.getparseexception()).getstatuscode());         if (link.getparseexception() instanceof sockettimeoutexception)             builder.append("time out");         if (link.getparseexception() instanceof malformedurlexception)             builder.append("url not valid");         if (link.getparseexception() instanceof unsupportedmimetypeexception)             builder.append("unsupported mime type: " + ((unsupportedmimetypeexception)link.getparseexception()).getmimetype());     }     return builder.tostring();  }  /**  * generates header file  * @param link  * @return  */ private string generateheaderfile(){     stringbuilder builder = new stringbuilder();     builder.append("seed url");     builder.append(",");     builder.append("seed ip");     builder.append(",");     builder.append("process duration");     builder.append(",");     builder.append("link url");     builder.append(",");     builder.append("link domain");     builder.append(",");     builder.append("link ip");     builder.append(",");     builder.append("enque time");     builder.append(",");     builder.append("deque time");     builder.append(",");     builder.append("waiting in queue");     builder.append(",");     builder.append("queuesize");     builder.append(",");     builder.append("intra in queue");     builder.append(",");     builder.append("inter in queue");     builder.append(",");     builder.append("dublications skipped");     /* time printed, no header */     builder.append(",");     builder.append("time");     /* url size*/     builder.append(",");     builder.append("size bytes");     /* http errors */     builder.append(",");     builder.append("http error");     return builder.tostring();  }    string geturlgeneralform(linknodelight link){     string url = link.geturl();     if (url.endswith("/")){         url = url.substring(0, url.length() - 1);     }     return url; }   private list<linknodelight> parseandwieghtresults(linknode inputlink) {     list<linknodelight> outputlinks = htmlparser.parse(inputlink);     if (inputlink.hasparseexception()) {         return outputlinks;     } else {         return urlweight.weight(inputlink, outputlinks);     } } } 

htmlparser.java

package pkg.crawler;  import org.jsoup.connection; import org.jsoup.jsoup; import org.jsoup.nodes.document; import org.jsoup.nodes.element; import org.jsoup.select.elements;  import java.io.bufferedwriter; import java.io.file; import java.io.fileoutputstream; import java.io.filewriter; import java.io.ioexception; import java.io.outputstreamwriter; import java.io.printwriter; import java.io.writer; import java.math.biginteger; import java.util.formatter; import java.util.hashmap; import java.util.linkedlist; import java.util.list; import java.util.concurrent.timeunit; import java.util.logging.logger; import java.security.*; import java.nio.file.path; import java.nio.file.paths;   public class htmlparser {  private static final int read_timeout_in_millissecs = (int) timeunit.milliseconds.convert(30, timeunit.seconds); private static hashmap <string, integer> filecounter = new hashmap<> ();   public static list<linknodelight> parse(linknode inputlink){     list<linknodelight> outputlinks = new linkedlist<>();     try {         inputlink.setipadress(ipfromurl.getip(inputlink.geturl()));         string url = inputlink.geturl();         if (inputlink.getipadress() != null) {             url.replace(urlweight.gethostname(url), inputlink.getipadress());         }         document parsedresults =  jsoup                 .connect(url)                 .timeout(read_timeout_in_millissecs)                 .get();         inputlink.setsize(parsedresults.html().length());         /* ip address moved here in order speed process */         inputlink.setstatus(linknodestatus.ok);         inputlink.setdomain(urlweight.getdomainname(inputlink.geturl()));         if (true) {             /* save file html */             string filename = parsedresults.title();//digestbig.tostring(16) + ".html";             if (filename.length() > 24) {                 filename = filename.substring(0, 24);             }             filename = filename.replaceall("[^\\w\\d\\s]", "").trim();             filename = filename.replaceall("\\s+",  " ");              if (!filecounter.containskey(filename)) {                 filecounter.put(filename, 1);             } else {                 integer tmp = filecounter.remove(filename);                 filecounter.put(filename, tmp + 1);             }             filename = filename + "-" + (filecounter.get(filename)).tostring() + ".html";             filename = paths.get("downloads", filename).tostring();             inputlink.setfilename(filename);             /* use md5 of url file name */             try (printwriter out = new printwriter(new bufferedwriter(new filewriter(filename)))) {                 out.println("<!--" + inputlink.geturl() + "-->");                 out.print(parsedresults.html());                 out.flush();                 out.close();             } catch (ioexception e) {                 e.printstacktrace();             }         }         string tag;         elements tagelements;         list<linknode> result;           tag = "a[href";         tagelements = parsedresults.select(tag);         result = tolinknodeobject(inputlink, tagelements, tag);         outputlinks.addall(result);           tag = "area[href";         tagelements = parsedresults.select(tag);         result = tolinknodeobject(inputlink, tagelements, tag);         outputlinks.addall(result);     } catch (ioexception e) {         inputlink.setparseexception(e);         inputlink.setstatus(linknodestatus.error);     }      return outputlinks; }   static list<linknode> tolinknodeobject(linknode parentlink, elements tagelements, string tag) {     list<linknode> links = new linkedlist<>();     (element element : tagelements) {          if(isfragmentref(element)){             continue;         }          string absoluteref = string.format("abs:%s", tag.contains("[") ? tag.substring(tag.indexof("[") + 1, tag.length()) : "href");         string url = element.attr(absoluteref);          if(url!=null && url.trim().length()>0) {             linknode link = new linknode(url);             link.settag(element.tagname());             link.setparentlink(parentlink);             links.add(link);         }     }     return links; }  static boolean isfragmentref(element element){     string href = element.attr("href");     return href!=null && (href.trim().startswith("#") || href.startswith("mailto:")); } 

}

util.java

package pkg.crawler;  import java.util.date;  import org.joda.time.datetime; import org.joda.time.format.datetimeformat; import org.joda.time.format.datetimeformatter;   public class util {  private static datetimeformatter formatter; static {        formatter =   datetimeformat.forpattern("yyyy-mm-dd hh:mm:ss:sss");   }   public static string linktostring(linknode inputlink){       return string.format("%s\t%s\t%s\t%s\t%s\t%s",             inputlink.geturl(),             inputlink.getweight(),             formatdate(inputlink.getenquetime()),             formatdate(inputlink.getdequetime()),             differenceinmilliseconds(inputlink.getenquetime(), inputlink.getdequetime()),             inputlink.getparentlink()==null?"":inputlink.getparentlink().geturl()     ); }  public static string linktoerrorstring(linknode inputlink){      return string.format("%s\t%s\t%s\t%s\t%s\t%s",             inputlink.geturl(),             inputlink.getweight(),             formatdate(inputlink.getenquetime()),             formatdate(inputlink.getdequetime()),             inputlink.getparentlink()==null?"":inputlink.getparentlink().geturl(),             inputlink.getparseexception().getmessage()     ); }   public static string formatdate(datetime date){     return formatter.print(date); }  public static long differenceinmilliseconds(datetime dequetime, datetime enquetime){     return (dequetime.getmillis()- enquetime.getmillis()); }  public static int differenceinseconds(date enquetime, date dequetime){     return (int)((dequetime.gettime()/1000) - (enquetime.gettime()/1000)); }  public static int differenceinminutes(date enquetime, date dequetime){     return (int)((dequetime.gettime()/60000) - (enquetime.gettime()/60000)); }  } 

urlweight.java

package pkg.crawler;  import java.util.arraylist; import java.util.hashset; import java.util.linkedlist; import java.util.list; import java.util.regex.pattern;  public class urlweight {  public static list<linknodelight> weight(linknode sourcelink, list<linknodelight> links) {      list<linknodelight> interlinks = new linkedlist<>();     list<linknodelight> intralinks = new linkedlist<>();      (linknodelight link : links) {         if (isintralink(sourcelink, link)) {             intralinks.add(link);             link.setinterlinks(false);         } else {             interlinks.add(link);             link.setinterlinks(true);         }     }    static boolean isintralink(linknodelight sourcelink, linknodelight link){      string parentdomainname = gethostname(sourcelink.geturl());      string childdomainname = gethostname(link.geturl());     return parentdomainname.equalsignorecase(childdomainname); }  public static string gethostname(string url) {     if(url == null){     //  system.out.println("deneme");         return "";      }      string domainname = new string(url);      int index = domainname.indexof("://");     if (index != -1) {          domainname = domainname.substring(index + 3);     }     (int = 0; < domainname.length(); i++)         if (domainname.charat(i) == '?' || domainname.charat(i) == '/') {             domainname = domainname.substring(0, i);             break;         }      /*if (index != -1) {          domainname = domainname.substring(0, index);     }*/      /* have keep www in order replacements ip */     //domainname = domainname.replacefirst("^www.*?\\.", "");      return domainname; } public static string getdomainname(string url) {     string [] tmp= gethostname(url).split("\\.");     if (tmp.length == 0)         return "";     return tmp[tmp.length - 1]; }   } 

pingtaskmanager.java

package pkg.crawler;  import java.util.concurrent.executorservice; import java.util.concurrent.executors;  public class pingtaskmanager {  private static executorservice executor = executors.newfixedthreadpool(100);  public  static void ping (linknode e) {     executor.submit(new pingtaks(e)); }   }  class pingtaks implements runnable {  private linknode link; public pingtaks( linknode link ) {  }  @override public void run() {     /* link.ping(); */       }   } 

linknodestatus.java

package pkg.crawler;  public enum linknodestatus { ok, error  } 

linknodelight.java

package pkg.crawler;  import org.joda.time.datetime;  public class linknodelight implements comparable<linknodelight> { protected string url; protected float weight; protected datetime enquetime; protected boolean interlinks;  public string geturl() {     return url; }  public float getweight() {     return weight; }  public void setweight(float weight) {     this.weight = weight; }  public datetime getenquetime() {     return enquetime; }   public linknodelight(string url) {     this.url = url; }   public void setenquetime(datetime enquetime) {     this.enquetime = enquetime; }  @override public int compareto(linknodelight link) {      if (this.weight < link.weight) return 1;      else if (this.weight > link.weight) return -1;         return 0;      } } 

linknode.java

package pkg.crawler;  import java.io.ioexception; import java.net.httpurlconnection; import java.net.socket; import java.net.url; import java.net.unknownhostexception; import java.util.date;    import org.joda.time.datetime;   public class linknode extends linknodelight{ public linknode(string url) {     super(url); }  private string tag; private linknode parentlink; private ioexception parseexception = null; // initialize parse exception null private float weight; private datetime dequetime; private datetime starttime; private datetime endtime; private linknodestatus status; private string ipadress; private int size; private string filename; private string domain;  public datetime getstarttime() {     return starttime; }  public void setstarttime(datetime starttime) {     this.starttime = starttime; }  public datetime getendtime() {     return endtime; }  public void setendtime(datetime endtime) {     this.endtime = endtime; }  public datetime getdequetime() {     return dequetime; }  public string gettag() {     return tag; }  public linknode getparentlink() {     return parentlink; }  public exception getparseexception() {     return parseexception; }  public boolean hasparseexception(){     return parseexception!=null; }   public void setdequetime(datetime dequetime) {     this.dequetime = dequetime; }  public void settag(string tag) {     this.tag = tag; }  public void setparentlink(linknode parentlink) {     this.parentlink = parentlink; }  public void setparseexception(ioexception parseexception) {     this.parseexception = parseexception; }  @override public boolean equals(object o) {     if (this == o) {         return true;     }     if (o == null || getclass() != o.getclass()) {         return false;     }      linknode link = (linknode) o;      if (url != null ? !url.equals(link.url) : link.url != null) {         return false;     }      return true; }  @override public int hashcode() {     return url != null ? url.hashcode() : 0; }  public long waitinginqueue(){     return util.differenceinmilliseconds( dequetime,enquetime ); }  public long linkprocessingduration(){     return util.differenceinmilliseconds( endtime,starttime ); }  @override public string tostring() {     stringbuilder sb = new stringbuilder("linknode{");     sb.append("url='").append(url).append('\'');     sb.append(", score=").append(weight);     sb.append(", enquetime=").append(enquetime);     sb.append(", dequetime=").append(dequetime);     sb.append(", tag=").append(tag);     if(parentlink!=null) {         sb.append(", parentlink=").append(parentlink.geturl());     }     sb.append('}');     return sb.tostring(); }  public void setstatus(linknodestatus status) {     this.status = status; }  public linknodestatus getstatus(){     if (status == null) {         status = linknodestatus.error;     }     return status; }  // check server link exist or not /* method gives fake errors public linknodestatus ping () {      boolean reachable = false;     string sanitizeurl = url.replacefirst("^https", "http");      try {         httpurlconnection connection = (httpurlconnection) new url(sanitizeurl).openconnection();         connection.setconnecttimeout(1000);         connection.setrequestmethod("head");         int responsecode = connection.getresponsecode();         system.err.println(url + " " + responsecode);         reachable = (200 <= responsecode && responsecode <= 399);     } catch (ioexception exception) {     }     return reachable?linknodestatus.ok: linknodestatus.error; }*/   public string getipadress() {     return ipadress; }  public void setipadress(string ipadress) {     this.ipadress = ipadress; }  /* methods controlling url size */ public void setsize(int size) {     this.size = size; }  public int getsize() {     return this.size; }  public void setfilename(string filename) {     this.filename = filename; }  public string getfilename() {     return this.filename; }  public string getdomain() {     return domain; }  public void setdomain(string domain) {     this.domain = domain;     } } 

i tried allocate memory changing eclipse.ini setting 2048 mb of ram answered in topic still same errors after 3 hours or less.

i hate repeat myself(*), in eclipse.ini set memory eclipse, has nothing memory crawler.

when using command line, need start via java -xmx2g pkg.crawler.webcrawler.

when starting eclipse, need add -xmx2g run configuration ("vm arguments" rather "program arguments").

---

(*) link deleted question; requires reputation view.