diff --git a/Tagger/Tagger.jar b/Tagger/Tagger.jar index c9b1400..bc0e9b5 100644 Binary files a/Tagger/Tagger.jar and b/Tagger/Tagger.jar differ diff --git a/Tagger/src/PixivManager.java b/Tagger/src/PixivManager.java index 72ed9fd..cf8dc13 100644 --- a/Tagger/src/PixivManager.java +++ b/Tagger/src/PixivManager.java @@ -40,8 +40,26 @@ public class PixivManager { fwOutput = new FileWriter(outputTest,true); bwOutput = new BufferedWriter(fwOutput); + int imageCounter=0; + while (imageCounter=imageTag.pixiv_image_list.size()) { + break; + } + if (i!=0) { + downloadData.append(","); + } + downloadData.append(imageTag.pixiv_image_list.get(imageCounter)); + imageCounter++; + } + System.out.println("Sending request to server to download "+downloadData.toString()); + utils.downloadFileFromUrl("http://45.33.13.215/crawler/request.php?images="+downloadData.toString(), "temp_req.html"); + } + for (String s : imageTag.pixiv_image_list) { - String url = "https://api.proxycrawl.com/?token=ahDRaxo3KT2OOX2nQZQV9A&url=https://www.pixiv.net/en/artworks/"+s; + //String url = "https://api.proxycrawl.com/?token=ahDRaxo3KT2OOX2nQZQV9A&url=https://www.pixiv.net/en/artworks/"+s; + String url = "http://45.33.13.215/crawler/crawler/"+s+".html"; try { if (!new File("downloadedData/temp"+s+".html").exists()) { System.out.println("Starting download of "+url+" ..."); diff --git a/Tagger/src/utils.java b/Tagger/src/utils.java index e5c452d..28c1c02 100644 --- a/Tagger/src/utils.java +++ b/Tagger/src/utils.java @@ -232,7 +232,7 @@ public class utils { System.out.println(s+": "+connection.getHeaderFields().get(s)); }*/ //connection.setRequestMethod("GET"); - connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestProperty("Content-Type", "text/html"); connection.setRequestProperty("User-Agent", "Mozilla/5.0"); int response = connection.getResponseCode(); //System.out.println("Response: "+response); @@ -267,8 +267,10 @@ public class utils { System.out.println(s+": "+connection.getHeaderFields().get(s)); }*/ //connection.setRequestMethod("GET"); - connection.setRequestProperty("Content-Type", "application/json"); + //connection.setRequestProperty("Content-Type", "text/html"); connection.setRequestProperty("User-Agent", "Mozilla/5.0"); + //connection.setRequestProperty("Referer", "pixiv.net"); + HttpURLConnection.setFollowRedirects(true); try { ReadableByteChannel rbc = Channels.newChannel(connection.getInputStream()); FileOutputStream fos = new FileOutputStream(file);