PixivImageTagger in final form.

pull/1/head
sigonasr2 5 years ago
commit 1affa36254
  1. 13
      Tagger/.externalToolBuilders/New_Builder.launch
  2. 27
      Tagger/.project
  3. BIN
      Tagger/Tagger.jar
  4. 7
      Tagger/downloadedData/.gitignore
  5. 8
      Tagger/filters.txt
  6. 1
      Tagger/lib/.gitignore
  7. BIN
      Tagger/lib/commons-imaging-1.0-alpha1.jar
  8. BIN
      Tagger/lib/json-20190722.jar
  9. BIN
      Tagger/log.txt
  10. 35
      Tagger/src/FileChooser.java
  11. 26
      Tagger/src/FilterFiles.java
  12. 24
      Tagger/src/Filters.java
  13. 212
      Tagger/src/PixivManager.java
  14. 42
      Tagger/src/Tagger.java
  15. BIN
      Tagger/src/exiftool.exe
  16. 91
      Tagger/src/imageTag.java
  17. 20
      Tagger/src/projectBuilder.xml
  18. 365
      Tagger/src/utils.java

@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType">
<booleanAttribute key="org.eclipse.ant.ui.ATTR_TARGETS_UPDATED" value="true"/>
<booleanAttribute key="org.eclipse.ant.ui.DEFAULT_VM_INSTALL" value="false"/>
<booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="false"/>
<stringAttribute key="org.eclipse.jdt.launching.CLASSPATH_PROVIDER" value="org.eclipse.ant.ui.AntClasspathProvider"/>
<booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="true"/>
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="Tagger"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${workspace_loc:/Tagger/src/projectBuilder.xml}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS" value="full,incremental,"/>
<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_WORKING_DIRECTORY" value="${workspace_loc:/Tagger}"/>
</launchConfiguration>

@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>Tagger</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.ui.externaltools.ExternalToolBuilder</name>
<triggers>full,incremental,</triggers>
<arguments>
<dictionary>
<key>LaunchConfigHandle</key>
<value>&lt;project&gt;/.externalToolBuilders/New_Builder.launch</value>
</dictionary>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>

Binary file not shown.

@ -0,0 +1,7 @@
/temp10131499.html
/temp48528896.html
/temp51582658.html
/temp68277325.html
/temp76863890.html
/temp76890277.html
/temp76928198.html

@ -0,0 +1,8 @@
jpg
jpeg
tga
tif
tiff
png
bmp
gif

@ -0,0 +1 @@
/exiftool.exe

Binary file not shown.

Binary file not shown.

@ -0,0 +1,35 @@
import java.io.File;
import javax.swing.JFileChooser;
public class FileChooser {
public FileChooser() {
JFileChooser chooser = new JFileChooser();
chooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY);
int val = chooser.showOpenDialog(null);
FilterFiles filter = new FilterFiles();
if (val==JFileChooser.APPROVE_OPTION) {
File[] files = chooser.getSelectedFile().listFiles(filter);
for (File file : files) {
if (file.isFile()) {
String filename = file.getName();
String newName = ReplaceExtraBits(filename);
System.out.println("Adding to Pixiv Album: " + newName);
imageTag.pixiv_image_list.add(newName);
imageTag.pixiv_rawimage_list.add(file);
}
}
}
}
String ReplaceExtraBits(String filename) {
return filename
.replaceAll("\\..{1,}", "") //regex to replace file extensions
.replaceAll("\\(.{1,}\\)", "") //regex to remove weird trimead parenthesis
.replaceAll("_p.{1,}","") //Remove the p stuff from pixiv images.
.trim(); //trim whitespace
}
}

@ -0,0 +1,26 @@
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class FilterFiles implements FilenameFilter{
public boolean accept(File f, String path) {
if (!path.matches(".{1,}_p[0-9]{1,}.{1,}")) {
System.out.println(path+" is not a pixiv image!");
return false;
}
for (String filter : imageTag.filters.filters) {
if (path.toLowerCase().endsWith(filter.toLowerCase())) {
//System.out.println("Accepting "+path+" because of filter "+filter);
return true;
}
}
return false;
}
}

@ -0,0 +1,24 @@
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class Filters {
List<String> filters = new ArrayList<String>();
public Filters() throws IOException {
File f = new File("filters.txt");
FileReader fr = new FileReader(f);
BufferedReader br = new BufferedReader(fr);
String s = br.readLine();
while (s!=null) {
String newfilter = s.trim();
filters.add(newfilter);
System.out.println("Adding filter "+newfilter);
s = br.readLine();
}
br.close();
fr.close();
}
}

@ -0,0 +1,212 @@
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
import org.apache.commons.imaging.ImageReadException;
import org.json.JSONArray;
import org.json.JSONObject;
public class PixivManager {
public PixivManager() {
File folder = new File("downloadedData");
if (folder.exists()) {
for (File fff : folder.listFiles()) {
if (fff.isFile()) {
fff.delete();
}
}
}
folder.mkdirs();
File outputTest = new File("TAG_DATA.txt");
FileWriter fwOutput;
BufferedWriter bwOutput;
try {
fwOutput = new FileWriter(outputTest,true);
bwOutput = new BufferedWriter(fwOutput);
for (String s : imageTag.pixiv_image_list) {
String url = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id="+s;
try {
if (!new File("downloadedData/temp"+s+".html").exists()) {
System.out.println("Starting download of "+url+" ...");
utils.downloadFileFromUrl(url, "downloadedData/temp"+s+".html");
String[] data = utils.readFromFile("downloadedData/temp"+s+".html");
int scriptEndLine = 0;
while (scriptEndLine<data.length) {
if (data[scriptEndLine].contains("return Object.freeze(arg)")) {
break;
}
scriptEndLine++;
}
if (scriptEndLine==data.length) {
System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
System.out.println(" IMAGE "+s+" FAILED TO PARSE CORRECTLY! Something is messed up about the file!!");
System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
}
File finaldata = new File("finaltemp");
FileWriter fw;
try {
fw = new FileWriter(finaldata);
BufferedWriter bw = new BufferedWriter(fw);
int cutpos = data[scriptEndLine+1].indexOf("})(")+3;
if (cutpos<data[scriptEndLine+1].length()) {
bw.write(data[scriptEndLine+1].substring(cutpos,data[scriptEndLine+1].indexOf(");</script>")));
}
bw.close();
fw.close();
} catch (IOException e) {
e.printStackTrace();
}
JSONObject jsonData = utils.readJsonFromFile("finaltemp");
//System.out.println(Arrays.deepToString(JSONObject.getNames(jsonData.getJSONObject("preload"))));
//System.out.println(Arrays.deepToString(JSONObject.getNames(jsonData.getJSONObject("preload").getJSONObject("illust"))));
JSONArray tagsArray = jsonData.getJSONObject("preload").getJSONObject("illust").getJSONObject(s).getJSONObject("tags").getJSONArray("tags");
for (int i=0;i<tagsArray.length();i++) {
boolean hasEnglishTag=false;
JSONObject tag = tagsArray.getJSONObject(i);
String ENTag="";
//String romaji="";
/*if (tag.has("romaji") && !tag.isNull("romaji")) {
romaji = tag.getString("romaji");
}*/
if (tag.has("translation")) {
JSONObject translationObj = tag.getJSONObject("translation");
if (translationObj.has("en")) {
hasEnglishTag=true;
ENTag = translationObj.getString("en");
}
} else
if (tag.has("tag") && /*romaji.length()==0 &&*/ !tag.getString("tag").matches(".*\\p{InHiragana}.*")) {
hasEnglishTag=true;
ENTag = tag.getString("tag");
}
if (ENTag.replaceAll("\\?", "").trim().length()==0) {
ENTag="";
hasEnglishTag=false;
}
boolean tagSubmitted=false;
String insertedTag="";
if (hasEnglishTag && ENTag.length()>0) {
insertedTag = ENTag;
tagSubmitted=true;
} /*else
if (romaji.length()>0){
insertedTag = romaji;
tagSubmitted=true;
}*/
if (tagSubmitted) {
if (imageTag.tag_whitelist.size()==0 || imageTag.tag_whitelist.containsKey(insertedTag)) {
if (imageTag.taglist.containsKey(s)) {
List<String> tags = imageTag.taglist.get(s);
tags.add(insertedTag);
imageTag.taglist.put(s, tags);
} else {
List<String> tags = new ArrayList<String>();
tags.add(insertedTag);
imageTag.taglist.put(s,tags);
}
if (imageTag.tagCounter.containsKey(insertedTag)) {
imageTag.tagCounter.put(insertedTag,imageTag.tagCounter.get(insertedTag)+1);
} else {
imageTag.tagCounter.put(insertedTag,1);
}
}
}
}
String taglist = s+": <"+imageTag.taglist.get(s)+">";
System.out.println(taglist);
bwOutput.append(taglist);
bwOutput.newLine();
//jsonData.getJSONObject("preload").getJSONObject("illust").getJSONObject(s).getJSONObject("tags");
} else {
System.out.println("Skipping image "+s+" because it has already been processed.");
}
} catch (IOException e) {
e.printStackTrace();
}
/*org.apache.commons.io.FileUtils.copyURLToFile(new URL(
url
),temp);*/
}
/*for (String s : imageTag.taglist.keySet()) {
System.out.println(s+": <"+imageTag.taglist.get(s)+">");
}*/
bwOutput.close();
fwOutput.close();
} catch (IOException e1) {
e1.printStackTrace();
}
Map sorted = sortByValues(imageTag.tagCounter);
Set s = sorted.entrySet();
Iterator i = s.iterator();
File tagFile = new File("SORTED_TAGS.txt");
FileWriter fw;
try {
fw = new FileWriter(tagFile);
BufferedWriter bw = new BufferedWriter(fw);
while (i.hasNext()) {
Map.Entry m = (Map.Entry)i.next();
String key = (String)m.getKey();
Integer value = (Integer)m.getValue();
bw.write(key+" - "+value);
bw.newLine();
}
bw.close();
fw.close();
} catch (IOException e1) {
e1.printStackTrace();
}
if (imageTag.tag_whitelist.size()==0) {
System.out.println("whitelist.txt not found! No tagging will be done this time. TAG_DATA.txt populated.");
} else {
System.out.println("Tagging Images...");
for (int j=0;j<imageTag.pixiv_image_list.size();j++) {
try {
Tagger tags = new Tagger(imageTag.pixiv_rawimage_list.get(j),imageTag.taglist.get(imageTag.pixiv_image_list.get(j)));
} catch (ImageReadException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
//System.out.println("Tagged "+ss+" with "+tagString);
}
}
}
public static <K, V extends Comparable<V>> Map<K, V>
sortByValues(final Map<K, V> map) {
Comparator<K> valueComparator =
new Comparator<K>() {
public int compare(K k1, K k2) {
int compare =
-map.get(k1).compareTo(map.get(k2));
if (compare == 0)
return 1;
else
return compare;
}
};
Map<K, V> sortedByValues =
new TreeMap<K, V>(valueComparator);
sortedByValues.putAll(map);
return sortedByValues;
}
}

@ -0,0 +1,42 @@
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.List;
import org.apache.commons.imaging.ImageReadException;
import org.apache.commons.imaging.ImageWriteException;
import org.apache.commons.imaging.Imaging;
import org.apache.commons.imaging.common.ImageMetadata;
import org.apache.commons.imaging.formats.jpeg.JpegImageMetadata;
import org.apache.commons.imaging.formats.tiff.TiffField;
import org.apache.commons.imaging.formats.tiff.TiffImageMetadata;
import org.apache.commons.imaging.formats.tiff.constants.ExifTagConstants;
import org.apache.commons.imaging.formats.tiff.constants.MicrosoftTagConstants;
import org.apache.commons.imaging.formats.tiff.write.TiffOutputDirectory;
import org.apache.commons.imaging.formats.tiff.write.TiffOutputField;
import org.apache.commons.imaging.formats.tiff.write.TiffOutputSet;
public class Tagger {
public Tagger(File imagetoTag,List<String> tags) throws ImageReadException, IOException {
StringBuilder sb = new StringBuilder();
if (tags!=null) {
for (String tag : tags) {
if (sb.length()!=0) {
sb.append("; ");
}
sb.append(tag);
}
Process tool = Runtime.getRuntime().exec("tool.exe -exif:XPKeywords=\""+sb.toString()+"\" "+imagetoTag.getAbsolutePath()+" -overwrite_original_in_place -P");
BufferedReader stdInput = new BufferedReader(new
InputStreamReader(tool.getInputStream()));
String s = null;
while ((s = stdInput.readLine()) != null) {
System.out.println("Tagging "+imagetoTag.getName()+":"+s);
}
}
}
}

Binary file not shown.

@ -0,0 +1,91 @@
import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import javax.swing.JFileChooser;
public class imageTag {
public static Filters filters;
public static HashMap<String,Boolean> tag_whitelist = new HashMap<String,Boolean>();
public static List<String> pixiv_image_list = new ArrayList<String>();
public static List<File> pixiv_rawimage_list = new ArrayList<File>();
public static HashMap<String,List<String>> taglist = new HashMap<String,List<String>>();
public static Map<String,Integer> tagCounter = new TreeMap<String,Integer>();
public static void main(String[] args) {
ExtractExifTool();
try {
filters = new Filters();
} catch (IOException e) {
e.printStackTrace();
}
GetTagWhitelist();
FileChooser fc = new FileChooser();
PixivManager pm = new PixivManager();
System.out.println("Done!");
}
private static void GetTagWhitelist() {
File whitelist = new File("whitelist.txt");
if (whitelist.exists()) {
FileReader fr;
try {
fr = new FileReader(whitelist);
BufferedReader br = new BufferedReader(fr);
String s;
try {
s = br.readLine();
while (s!=null) {
String newtag = s.trim();
tag_whitelist.put(newtag,true);
System.out.println("Read in whitelisted tag: "+newtag);
s=br.readLine();
}
} catch (IOException e) {
e.printStackTrace();
}
} catch (FileNotFoundException e1) {
e1.printStackTrace();
}
}
}
private static void ExtractExifTool() {
InputStream tool = imageTag.class.getResourceAsStream("exiftool.exe");
File f = new File("tool.exe");
try {
byte[] buffer = new byte[tool.available()];
tool.read(buffer);
OutputStream outStream = new FileOutputStream(f);
outStream.write(buffer);
outStream.close();
tool.close();
f.deleteOnExit();
} catch (IOException e) {
e.printStackTrace();
}
}
}

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<project default="create_run_jar" name="Create Runnable Jar for Project sigIRCv2">
<!--this file was created by Eclipse Runnable JAR Export Wizard-->
<!--ANT 1.7 is required -->
<!--define folder properties-->
<property name="dir.buildfile" value="."/>
<property name="dir.workspace" value="${dir.buildfile}/.."/>
<property name="dir.jarfile" value="${dir.buildfile}"/>
<target name="create_run_jar">
<jar destfile="${dir.jarfile}/Tagger.jar" filesetmanifest="mergewithoutmain">
<manifest>
<attribute name="Main-Class" value="imageTag"/>
<attribute name="Class-Path" value="."/>
</manifest>
<fileset dir="${dir.jarfile}/bin"/>
<zipfileset excludes="META-INF/*.SF" src="lib/commons-imaging-1.0-alpha1.jar"/>
<zipfileset excludes="META-INF/*.SF" src="lib/json-20190722.jar"/>
</jar>
</target>
</project>

@ -0,0 +1,365 @@
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
public class utils {
public static String[] readFromFile(String filename) {
File file = new File(filename);
//System.out.println(file.getAbsolutePath());
List<String> contents= new ArrayList<String>();
if (file.exists()) {
try(
FileReader fw = new FileReader(filename);
BufferedReader bw = new BufferedReader(fw);)
{
String readline = bw.readLine();
do {
if (readline!=null) {
//System.out.println(readline);
contents.add(readline);
readline = bw.readLine();
}} while (readline!=null);
fw.close();
bw.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return contents.toArray(new String[contents.size()]);
}
private static String readAll(Reader rd) throws IOException {
StringBuilder sb = new StringBuilder();
int cp;
while ((cp = rd.read()) != -1) {
sb.append((char) cp);
}
return sb.toString();
}
private static String readFilter(Reader rd, HashMap<Long,String> channel_ids) throws IOException {
StringBuilder sb = new StringBuilder();
boolean allowed=false;
boolean quotation_mark=false;
boolean endquotation_mark=false;
boolean foundChannel=false;
boolean nextBrace=false;
boolean outputStuff=false;
String numb = "";
int braceCount=0;
int channelCount=0;
int vals=0;
int cp;
while ((cp = rd.read()) != -1) {
if (braceCount==0) {
allowed=true;
} else
if (braceCount==1 && !quotation_mark){
quotation_mark=true;
numb="";
allowed=false;
} else
if (!endquotation_mark) {
if ((char)cp >= '0' &&
(char)cp <= '9') {
allowed=false;
numb+=(char)cp;
} else {
allowed=false;
endquotation_mark=true;
try {
if (channel_ids.containsKey(Long.parseLong(numb))) {
if (channelCount>=1) {
sb.append(",");
}
sb.append("\""+numb+"\"");
foundChannel=true;
System.out.println("Found channel "+numb);
outputStuff=true;
}
} catch (NumberFormatException e) {
}
}
} else
if (!nextBrace && foundChannel) {
allowed=true;
if ((char)cp == '{') {
nextBrace=true;
}
} else
if (foundChannel) {
allowed=true;
if (braceCount==1) {
allowed=false;
channelCount++;
quotation_mark=false;
endquotation_mark=false;
foundChannel=false;
nextBrace=false;
}
} else {
allowed=false;
if (braceCount==1) {
allowed=false;
quotation_mark=false;
endquotation_mark=false;
foundChannel=false;
nextBrace=false;
}
}
/*if (outputStuff && vals++<1000) {
System.out.print((char)cp);
}*/
if ((char)cp == '{') {
braceCount++;
//System.out.println("Brace count is "+braceCount+".");
} else
if ((char)cp == '}') {
braceCount--;
//System.out.println("Brace count is "+braceCount+".");
}
if (allowed) {
sb.append((char) cp);
}
}
sb.append("}");
//System.out.println("=============");
//System.out.println(sb.toString());
return sb.toString();
}
public static JSONObject readJsonFromUrlWithFilter(String url, HashMap<Long,String> filter) throws IOException, JSONException {
return readJsonFromUrlWithFilter(url,filter,null,false);
}
public static JSONObject readJsonFromFileWithFilter(String file, HashMap<Long,String> filter) throws IOException, JSONException {
InputStream is = new FileInputStream(new File(file));
try {
BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
String jsonText = readFilter(rd,filter);
JSONObject json = new JSONObject(jsonText);
jsonText=null;
return json;
} finally {
is.close();
}
}
public static JSONObject readJsonFromUrlWithFilter(String url, HashMap<Long,String> filter, String file, boolean writeToFile) throws IOException, JSONException {
InputStream is = new URL(url).openStream();
try {
BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
String jsonText = readFilter(rd,filter);
if (writeToFile) {
writetoFile(new String[]{jsonText},file);
}
JSONObject json = new JSONObject(jsonText);
return json;
} finally {
is.close();
}
}
public static JSONObject readJsonFromUrl(String url) throws IOException, JSONException {
return readJsonFromUrl(url,null,false);
}
public static JSONArray readJsonArrayFromUrl(String url) throws IOException, JSONException {
return readJsonArrayFromUrl(url,null,false);
}
public static JSONObject readJsonFromFile(String file) throws IOException, JSONException {
InputStream is = new FileInputStream(new File(file));
try {
BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
String jsonText = readAll(rd);
JSONObject json = new JSONObject(jsonText);
jsonText=null;
return json;
} finally {
is.close();
}
}
public static JSONObject readJsonFromFile(File file) throws IOException, JSONException {
InputStream is = new FileInputStream(file);
try {
BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
String jsonText = readAll(rd);
JSONObject json = new JSONObject(jsonText);
jsonText=null;
return json;
} finally {
is.close();
}
}
public static JSONObject readJsonFromUrl(String url, String file, boolean writeToFile) throws IOException, JSONException {
//InputStream is = new URL(url).openStream();
URL site = new URL(url);
HttpURLConnection connection = (HttpURLConnection) site.openConnection();
/*for (String s : connection.getHeaderFields().keySet()) {
System.out.println(s+": "+connection.getHeaderFields().get(s));
}*/
//connection.setRequestMethod("GET");
connection.setRequestProperty("Content-Type", "application/json");
connection.setRequestProperty("User-Agent", "Mozilla/5.0");
int response = connection.getResponseCode();
//System.out.println("Response: "+response);
InputStreamReader stream = new InputStreamReader(connection.getInputStream());
BufferedReader rd = new BufferedReader(stream);
String jsonText = readAll(rd);
if (writeToFile) {
writetoFile(new String[]{jsonText},file);
}
JSONObject json = new JSONObject(jsonText);
stream.close();
return json;
}
static int LastSlash(String s) {
int lastSlashpos = 0;
for (int i=0;i<s.length();i++) {
if (s.charAt(i)=='/') {
lastSlashpos=i;
}
}
return lastSlashpos;
}
public static void downloadFileFromUrl(String url, String file) throws IOException, JSONException {
String temp = url.substring(0,LastSlash(url));
String temp2 = url.substring(LastSlash(url));
URL website = new URL(url);
HttpURLConnection connection = (HttpURLConnection) website.openConnection();
/*for (String s : connection.getHeaderFields().keySet()) {
System.out.println(s+": "+connection.getHeaderFields().get(s));
}*/
//connection.setRequestMethod("GET");
connection.setRequestProperty("Content-Type", "application/json");
connection.setRequestProperty("User-Agent", "Mozilla/5.0");
ReadableByteChannel rbc = Channels.newChannel(connection.getInputStream());
FileOutputStream fos = new FileOutputStream(file);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
}
public static JSONArray readJsonArrayFromUrl(String url, String file, boolean writeToFile) throws IOException, JSONException {
//InputStream is = new URL(url).openStream();
URL site = new URL(url);
HttpURLConnection connection = (HttpURLConnection) site.openConnection();
/*for (String s : connection.getHeaderFields().keySet()) {
System.out.println(s+": "+connection.getHeaderFields().get(s));
}*/
//connection.setRequestMethod("GET");
connection.setRequestProperty("Content-Type", "application/json");
connection.setRequestProperty("User-Agent", "Mozilla/5.0");
int response = connection.getResponseCode();
//System.out.println("Response: "+response);
InputStreamReader stream = new InputStreamReader(connection.getInputStream());
BufferedReader rd = new BufferedReader(stream);
String jsonText = readAll(rd);
if (writeToFile) {
writetoFile(new String[]{jsonText},file);
}
JSONArray json = new JSONArray(jsonText);
stream.close();
return json;
}
/*public static void logToFile(String message, String filename) {
logToFile(message,filename,false);
}*/
/*public static void logToFile(String message, String filename, boolean outputToChatLog) {
File file = new File(filename);
try {
if (!file.exists()) {
file.createNewFile();
}
FileWriter fw = new FileWriter(file, true);
PrintWriter pw = new PrintWriter(fw);
pw.println(message);
pw.flush();
pw.close();
} catch (IOException e) {
e.printStackTrace();
}
if (outputToChatLog && sigIRC.chatlogmodule_enabled) {
ChatLogMessage.importMessages(message);
}
}*/
public static void writetoFile(String[] data, String filename) {
File file = new File(filename);
try {
if (!file.exists()) {
file.createNewFile();
}
FileWriter fw = new FileWriter(file,false);
PrintWriter pw = new PrintWriter(fw);
for (String s : data) {
pw.println(s);
}
pw.flush();
pw.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public static void copyFile(File source, File dest) throws IOException {
FileChannel sourceChannel = null;
FileChannel destChannel = null;
try {
sourceChannel = new FileInputStream(source).getChannel();
destChannel = new FileOutputStream(dest).getChannel();
destChannel.transferFrom(sourceChannel, 0, sourceChannel.size());
}finally{
sourceChannel.close();
destChannel.close();
}
}
public static void deleteFile(String filename) {
File file = new File(filename);
if (file.exists()) {
file.delete();
}
}
}
Loading…
Cancel
Save