Change web crawler to point to Sig's server. Implemented server fetch /

request system.

Tagger/src/PixivManager.java

@@ -40,8 +40,26 @@ public class PixivManager {
 			fwOutput = new FileWriter(outputTest,true);
 			bwOutput = new BufferedWriter(fwOutput);
 			
+			int imageCounter=0;
+			while (imageCounter<imageTag.pixiv_image_list.size()) {
+				StringBuilder downloadData = new StringBuilder();
+				for (int i=0;i<20;i++) {
+					if (imageCounter>=imageTag.pixiv_image_list.size()) {
+						break;
+					}
+					if (i!=0) {
+						downloadData.append(",");
+					}
+					downloadData.append(imageTag.pixiv_image_list.get(imageCounter));
+					imageCounter++;
+				}
+				System.out.println("Sending request to server to download "+downloadData.toString());
+				utils.downloadFileFromUrl("http://45.33.13.215/crawler/request.php?images="+downloadData.toString(), "temp_req.html");
+			}
+			
 			for (String s : imageTag.pixiv_image_list) {
-				String url = "https://api.proxycrawl.com/?token=ahDRaxo3KT2OOX2nQZQV9A&url=https://www.pixiv.net/en/artworks/"+s;
+				//String url = "https://api.proxycrawl.com/?token=ahDRaxo3KT2OOX2nQZQV9A&url=https://www.pixiv.net/en/artworks/"+s;
+				String url = "http://45.33.13.215/crawler/crawler/"+s+".html";
 				try {
 					if (!new File("downloadedData/temp"+s+".html").exists()) {
 						System.out.println("Starting download of "+url+" ...");

Tagger/src/utils.java

@@ -232,7 +232,7 @@ public class utils {
 		    	System.out.println(s+": "+connection.getHeaderFields().get(s));
 		    }*/
 		    //connection.setRequestMethod("GET");
-		    connection.setRequestProperty("Content-Type", "application/json");
+		    connection.setRequestProperty("Content-Type", "text/html");
 		    connection.setRequestProperty("User-Agent", "Mozilla/5.0");
 		    int response = connection.getResponseCode();
 		    //System.out.println("Response: "+response);
@@ -267,8 +267,10 @@ public class utils {
 		    	System.out.println(s+": "+connection.getHeaderFields().get(s));
 		    }*/
 		    //connection.setRequestMethod("GET");
-		    connection.setRequestProperty("Content-Type", "application/json");
+		    //connection.setRequestProperty("Content-Type", "text/html");
 		    connection.setRequestProperty("User-Agent", "Mozilla/5.0");
+		    //connection.setRequestProperty("Referer", "pixiv.net");
+		    HttpURLConnection.setFollowRedirects(true);
 		    try {
 			  ReadableByteChannel rbc = Channels.newChannel(connection.getInputStream());
 			  FileOutputStream fos = new FileOutputStream(file);