sigonasr2
/
pixivImageTagger
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

PixivImageTagger in final form.

Browse Source
pull/1/head
sigonasr2 5 years ago
commit 1affa36254
18 changed files with 871 additions and 0 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 13
      Tagger/.externalToolBuilders/New_Builder.launch
  2. 27
      Tagger/.project
  3. BIN
      Tagger/Tagger.jar
  4. 7
      Tagger/downloadedData/.gitignore
  5. 8
      Tagger/filters.txt
  6. 1
      Tagger/lib/.gitignore
  7. BIN
      Tagger/lib/commons-imaging-1.0-alpha1.jar
  8. BIN
      Tagger/lib/json-20190722.jar
  9. BIN
      Tagger/log.txt
  10. 35
      Tagger/src/FileChooser.java
  11. 26
      Tagger/src/FilterFiles.java
  12. 24
      Tagger/src/Filters.java
  13. 212
      Tagger/src/PixivManager.java
  14. 42
      Tagger/src/Tagger.java
  15. BIN
      Tagger/src/exiftool.exe
  16. 91
      Tagger/src/imageTag.java
  17. 20
      Tagger/src/projectBuilder.xml
  18. 365
      Tagger/src/utils.java

13
Tagger/.externalToolBuilders/New_Builder.launch
Unescape View File

@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType">
<booleanAttribute key="org.eclipse.ant.ui.ATTR_TARGETS_UPDATED" value="true"/>
<booleanAttribute key="org.eclipse.ant.ui.DEFAULT_VM_INSTALL" value="false"/>
<booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="false"/>
<stringAttribute key="org.eclipse.jdt.launching.CLASSPATH_PROVIDER" value="org.eclipse.ant.ui.AntClasspathProvider"/>
<booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="true"/>
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="Tagger"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${workspace_loc:/Tagger/src/projectBuilder.xml}"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS" value="full,incremental,"/>
<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/>
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_WORKING_DIRECTORY" value="${workspace_loc:/Tagger}"/>
</launchConfiguration>

27
Tagger/.project
Unescape View File

@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>Tagger</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.ui.externaltools.ExternalToolBuilder</name>
<triggers>full,incremental,</triggers>
<arguments>
<dictionary>
<key>LaunchConfigHandle</key>
<value>&lt;project&gt;/.externalToolBuilders/New_Builder.launch</value>
</dictionary>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>

BIN
Tagger/Tagger.jar
View File

Binary file not shown.

7
Tagger/downloadedData/.gitignore vendored
Unescape View File

@ -0,0 +1,7 @@
/temp10131499.html
/temp48528896.html
/temp51582658.html
/temp68277325.html
/temp76863890.html
/temp76890277.html
/temp76928198.html

8
Tagger/filters.txt
Unescape View File

@ -0,0 +1,8 @@
jpg
jpeg
tga
tif
tiff
png
bmp
gif

1
Tagger/lib/.gitignore vendored
Unescape View File

@ -0,0 +1 @@
/exiftool.exe

BIN
Tagger/lib/commons-imaging-1.0-alpha1.jar
View File

Binary file not shown.

BIN
Tagger/lib/json-20190722.jar
View File

Binary file not shown.

BIN
Tagger/log.txt
View File

Binary file not shown.

35
Tagger/src/FileChooser.java
Unescape View File

@ -0,0 +1,35 @@
import java.io.File;
import javax.swing.JFileChooser;
public class FileChooser {
public FileChooser() {
JFileChooser chooser = new JFileChooser();
chooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY);
int val = chooser.showOpenDialog(null);
FilterFiles filter = new FilterFiles();
if (val==JFileChooser.APPROVE_OPTION) {
File[] files = chooser.getSelectedFile().listFiles(filter);
for (File file : files) {
if (file.isFile()) {
String filename = file.getName();
String newName = ReplaceExtraBits(filename);
System.out.println("Adding to Pixiv Album: " + newName);
imageTag.pixiv_image_list.add(newName);
imageTag.pixiv_rawimage_list.add(file);
}
}
}
}
String ReplaceExtraBits(String filename) {
return filename
.replaceAll("\\..{1,}", "") //regex to replace file extensions
.replaceAll("\\(.{1,}\\)", "") //regex to remove weird trimead parenthesis
.replaceAll("_p.{1,}","") //Remove the p stuff from pixiv images.
.trim(); //trim whitespace
}
}

26
Tagger/src/FilterFiles.java
Unescape View File

@ -0,0 +1,26 @@
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class FilterFiles implements FilenameFilter{
public boolean accept(File f, String path) {
if (!path.matches(".{1,}_p[0-9]{1,}.{1,}")) {
System.out.println(path+" is not a pixiv image!");
return false;
}
for (String filter : imageTag.filters.filters) {
if (path.toLowerCase().endsWith(filter.toLowerCase())) {
//System.out.println("Accepting "+path+" because of filter "+filter);
return true;
}
}
return false;
}
}

24
Tagger/src/Filters.java
Unescape View File

@ -0,0 +1,24 @@
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class Filters {
List<String> filters = new ArrayList<String>();
public Filters() throws IOException {
File f = new File("filters.txt");
FileReader fr = new FileReader(f);
BufferedReader br = new BufferedReader(fr);
String s = br.readLine();
while (s!=null) {
String newfilter = s.trim();
filters.add(newfilter);
System.out.println("Adding filter "+newfilter);
s = br.readLine();
}
br.close();
fr.close();
}
}

212
Tagger/src/PixivManager.java
Unescape View File

@ -0,0 +1,212 @@
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
import org.apache.commons.imaging.ImageReadException;
import org.json.JSONArray;
import org.json.JSONObject;
public class PixivManager {
public PixivManager() {
File folder = new File("downloadedData");
if (folder.exists()) {
for (File fff : folder.listFiles()) {
if (fff.isFile()) {
fff.delete();
}
}
}
folder.mkdirs();
File outputTest = new File("TAG_DATA.txt");
FileWriter fwOutput;
BufferedWriter bwOutput;
try {
fwOutput = new FileWriter(outputTest,true);
bwOutput = new BufferedWriter(fwOutput);
for (String s : imageTag.pixiv_image_list) {
String url = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id="+s;
try {
if (!new File("downloadedData/temp"+s+".html").exists()) {
System.out.println("Starting download of "+url+" ...");
utils.downloadFileFromUrl(url, "downloadedData/temp"+s+".html");
String[] data = utils.readFromFile("downloadedData/temp"+s+".html");
int scriptEndLine = 0;
while (scriptEndLine<data.length) {
if (data[scriptEndLine].contains("return Object.freeze(arg)")) {
break;
}
scriptEndLine++;
}
if (scriptEndLine==data.length) {
System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
System.out.println(" IMAGE "+s+" FAILED TO PARSE CORRECTLY! Something is messed up about the file!!");
System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
}
File finaldata = new File("finaltemp");
FileWriter fw;
try {
fw = new FileWriter(finaldata);
BufferedWriter bw = new BufferedWriter(fw);
int cutpos = data[scriptEndLine+1].indexOf("})(")+3;
if (cutpos<data[scriptEndLine+1].length()) {
bw.write(data[scriptEndLine+1].substring(cutpos,data[scriptEndLine+1].indexOf(");</script>")));
}
bw.close();
fw.close();
} catch (IOException e) {
e.printStackTrace();
}
JSONObject jsonData = utils.readJsonFromFile("finaltemp");
//System.out.println(Arrays.deepToString(JSONObject.getNames(jsonData.getJSONObject("preload"))));
//System.out.println(Arrays.deepToString(JSONObject.getNames(jsonData.getJSONObject("preload").getJSONObject("illust"))));
JSONArray tagsArray = jsonData.getJSONObject("preload").getJSONObject("illust").getJSONObject(s).getJSONObject("tags").getJSONArray("tags");
for (int i=0;i<tagsArray.length();i++) {
boolean hasEnglishTag=false;
JSONObject tag = tagsArray.getJSONObject(i);
String ENTag="";
//String romaji="";
/*if (tag.has("romaji") && !tag.isNull("romaji")) {
romaji = tag.getString("romaji");
}*/
if (tag.has("translation")) {
JSONObject translationObj = tag.getJSONObject("translation");
if (translationObj.has("en")) {
hasEnglishTag=true;
ENTag = translationObj.getString("en");
}
} else
if (tag.has("tag") && /*romaji.length()==0 &&*/ !tag.getString("tag").matches(".*\\p{InHiragana}.*")) {
hasEnglishTag=true;
ENTag = tag.getString("tag");
}
if (ENTag.replaceAll("\\?", "").trim().length()==0) {
ENTag="";
hasEnglishTag=false;
}
boolean tagSubmitted=false;
String insertedTag="";
if (hasEnglishTag && ENTag.length()>0) {
insertedTag = ENTag;
tagSubmitted=true;
} /*else
if (romaji.length()>0){
insertedTag = romaji;
tagSubmitted=true;
}*/
if (tagSubmitted) {
if (imageTag.tag_whitelist.size()==0 || imageTag.tag_whitelist.containsKey(insertedTag)) {
if (imageTag.taglist.containsKey(s)) {
List<String> tags = imageTag.taglist.get(s);
tags.add(insertedTag);
imageTag.taglist.put(s, tags);
} else {
List<String> tags = new ArrayList<String>();
tags.add(insertedTag);
imageTag.taglist.put(s,tags);
}
if (imageTag.tagCounter.containsKey(insertedTag)) {
imageTag.tagCounter.put(insertedTag,imageTag.tagCounter.get(insertedTag)+1);
} else {
imageTag.tagCounter.put(insertedTag,1);
}
}
}
}
String taglist = s+": <"+imageTag.taglist.get(s)+">";
System.out.println(taglist);
bwOutput.append(taglist);
bwOutput.newLine();
//jsonData.getJSONObject("preload").getJSONObject("illust").getJSONObject(s).getJSONObject("tags");
} else {
System.out.println("Skipping image "+s+" because it has already been processed.");
}
} catch (IOException e) {
e.printStackTrace();
}
/*org.apache.commons.io.FileUtils.copyURLToFile(new URL(
url
),temp);*/
}
/*for (String s : imageTag.taglist.keySet()) {
System.out.println(s+": <"+imageTag.taglist.get(s)+">");
}*/
bwOutput.close();
fwOutput.close();
} catch (IOException e1) {
e1.printStackTrace();
}
Map sorted = sortByValues(imageTag.tagCounter);
Set s = sorted.entrySet();
Iterator i = s.iterator();
File tagFile = new File("SORTED_TAGS.txt");
FileWriter fw;
try {
fw = new FileWriter(tagFile);
BufferedWriter bw = new BufferedWriter(fw);
while (i.hasNext()) {
Map.Entry m = (Map.Entry)i.next();
String key = (String)m.getKey();
Integer value = (Integer)m.getValue();
bw.write(key+" - "+value);
bw.newLine();
}
bw.close();
fw.close();
} catch (IOException e1) {
e1.printStackTrace();
}
if (imageTag.tag_whitelist.size()==0) {
System.out.println("whitelist.txt not found! No tagging will be done this time. TAG_DATA.txt populated.");
} else {
System.out.println("Tagging Images...");
for (int j=0;j<imageTag.pixiv_image_list.size();j++) {
try {
Tagger tags = new Tagger(imageTag.pixiv_rawimage_list.get(j),imageTag.taglist.get(imageTag.pixiv_image_list.get(j)));
} catch (ImageReadException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
//System.out.println("Tagged "+ss+" with "+tagString);
}
}
}
public static <K, V extends Comparable<V>> Map<K, V>
sortByValues(final Map<K, V> map) {
Comparator<K> valueComparator =
new Comparator<K>() {
public int compare(K k1, K k2) {
int compare =
-map.get(k1).compareTo(map.get(k2));
if (compare == 0)
return 1;
else
return compare;
}
};
Map<K, V> sortedByValues =
new TreeMap<K, V>(valueComparator);
sortedByValues.putAll(map);
return sortedByValues;
}
}

42
Tagger/src/Tagger.java
Unescape View File

@ -0,0 +1,42 @@
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.List;
import org.apache.commons.imaging.ImageReadException;
import org.apache.commons.imaging.ImageWriteException;
import org.apache.commons.imaging.Imaging;
import org.apache.commons.imaging.common.ImageMetadata;
import org.apache.commons.imaging.formats.jpeg.JpegImageMetadata;
import org.apache.commons.imaging.formats.tiff.TiffField;
import org.apache.commons.imaging.formats.tiff.TiffImageMetadata;
import org.apache.commons.imaging.formats.tiff.constants.ExifTagConstants;
import org.apache.commons.imaging.formats.tiff.constants.MicrosoftTagConstants;
import org.apache.commons.imaging.formats.tiff.write.TiffOutputDirectory;
import org.apache.commons.imaging.formats.tiff.write.TiffOutputField;
import org.apache.commons.imaging.formats.tiff.write.TiffOutputSet;
public class Tagger {
public Tagger(File imagetoTag,List<String> tags) throws ImageReadException, IOException {
StringBuilder sb = new StringBuilder();
if (tags!=null) {
for (String tag : tags) {
if (sb.length()!=0) {
sb.append("; ");
}
sb.append(tag);
}
Process tool = Runtime.getRuntime().exec("tool.exe -exif:XPKeywords=\""+sb.toString()+"\" "+imagetoTag.getAbsolutePath()+" -overwrite_original_in_place -P");
BufferedReader stdInput = new BufferedReader(new
InputStreamReader(tool.getInputStream()));
String s = null;
while ((s = stdInput.readLine()) != null) {
System.out.println("Tagging "+imagetoTag.getName()+":"+s);
}
}
}
}

BIN
Tagger/src/exiftool.exe
View File

Binary file not shown.

91
Tagger/src/imageTag.java
Unescape View File

@ -0,0 +1,91 @@
import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import javax.swing.JFileChooser;
public class imageTag {
public static Filters filters;
public static HashMap<String,Boolean> tag_whitelist = new HashMap<String,Boolean>();
public static List<String> pixiv_image_list = new ArrayList<String>();
public static List<File> pixiv_rawimage_list = new ArrayList<File>();
public static HashMap<String,List<String>> taglist = new HashMap<String,List<String>>();
public static Map<String,Integer> tagCounter = new TreeMap<String,Integer>();
public static void main(String[] args) {
ExtractExifTool();
try {
filters = new Filters();
} catch (IOException e) {
e.printStackTrace();
}
GetTagWhitelist();
FileChooser fc = new FileChooser();
PixivManager pm = new PixivManager();
System.out.println("Done!");
}
private static void GetTagWhitelist() {
File whitelist = new File("whitelist.txt");
if (whitelist.exists()) {
FileReader fr;
try {
fr = new FileReader(whitelist);
BufferedReader br = new BufferedReader(fr);
String s;
try {
s = br.readLine();
while (s!=null) {
String newtag = s.trim();
tag_whitelist.put(newtag,true);
System.out.println("Read in whitelisted tag: "+newtag);
s=br.readLine();
}
} catch (IOException e) {
e.printStackTrace();
}
} catch (FileNotFoundException e1) {
e1.printStackTrace();
}
}
}
private static void ExtractExifTool() {
InputStream tool = imageTag.class.getResourceAsStream("exiftool.exe");
File f = new File("tool.exe");
try {
byte[] buffer = new byte[tool.available()];
tool.read(buffer);
OutputStream outStream = new FileOutputStream(f);
outStream.write(buffer);
outStream.close();
tool.close();
f.deleteOnExit();
} catch (IOException e) {
e.printStackTrace();
}
}
}

20
Tagger/src/projectBuilder.xml
Unescape View File

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<project default="create_run_jar" name="Create Runnable Jar for Project sigIRCv2">
<!--this file was created by Eclipse Runnable JAR Export Wizard-->
<!--ANT 1.7 is required -->
<!--define folder properties-->
<property name="dir.buildfile" value="."/>
<property name="dir.workspace" value="${dir.buildfile}/.."/>
<property name="dir.jarfile" value="${dir.buildfile}"/>
<target name="create_run_jar">
<jar destfile="${dir.jarfile}/Tagger.jar" filesetmanifest="mergewithoutmain">
<manifest>
<attribute name="Main-Class" value="imageTag"/>
<attribute name="Class-Path" value="."/>
</manifest>
<fileset dir="${dir.jarfile}/bin"/>
<zipfileset excludes="META-INF/*.SF" src="lib/commons-imaging-1.0-alpha1.jar"/>
<zipfileset excludes="META-INF/*.SF" src="lib/json-20190722.jar"/>
</jar>
</target>
</project>

365
Tagger/src/utils.java
Unescape View File

@ -0,0 +1,365 @@
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
public class utils {
public static String[] readFromFile(String filename) {
File file = new File(filename);
//System.out.println(file.getAbsolutePath());
List<String> contents= new ArrayList<String>();
if (file.exists()) {
try(
FileReader fw = new FileReader(filename);
BufferedReader bw = new BufferedReader(fw);)
{
String readline = bw.readLine();
do {
if (readline!=null) {
//System.out.println(readline);
contents.add(readline);
readline = bw.readLine();
}} while (readline!=null);
fw.close();
bw.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return contents.toArray(new String[contents.size()]);
}
private static String readAll(Reader rd) throws IOException {
StringBuilder sb = new StringBuilder();
int cp;
while ((cp = rd.read()) != -1) {
sb.append((char) cp);
}
return sb.toString();
}
private static String readFilter(Reader rd, HashMap<Long,String> channel_ids) throws IOException {
StringBuilder sb = new StringBuilder();
boolean allowed=false;
boolean quotation_mark=false;
boolean endquotation_mark=false;
boolean foundChannel=false;
boolean nextBrace=false;
boolean outputStuff=false;
String numb = "";
int braceCount=0;
int channelCount=0;
int vals=0;
int cp;
while ((cp = rd.read()) != -1) {
if (braceCount==0) {
allowed=true;
} else
if (braceCount==1 && !quotation_mark){
quotation_mark=true;
numb="";
allowed=false;
} else
if (!endquotation_mark) {
if ((char)cp >= '0' &&
(char)cp <= '9') {
allowed=false;
numb+=(char)cp;
} else {
allowed=false;
endquotation_mark=true;
try {
if (channel_ids.containsKey(Long.parseLong(numb))) {
if (channelCount>=1) {
sb.append(",");
}
sb.append("\""+numb+"\"");
foundChannel=true;
System.out.println("Found channel "+numb);
outputStuff=true;
}
} catch (NumberFormatException e) {
}
}
} else
if (!nextBrace && foundChannel) {
allowed=true;
if ((char)cp == '{') {
nextBrace=true;
}
} else
if (foundChannel) {
allowed=true;
if (braceCount==1) {
allowed=false;
channelCount++;
quotation_mark=false;
endquotation_mark=false;
foundChannel=false;
nextBrace=false;
}
} else {
allowed=false;
if (braceCount==1) {
allowed=false;
quotation_mark=false;
endquotation_mark=false;
foundChannel=false;
nextBrace=false;
}
}
/*if (outputStuff && vals++<1000) {
System.out.print((char)cp);
}*/
if ((char)cp == '{') {
braceCount++;
//System.out.println("Brace count is "+braceCount+".");
} else
if ((char)cp == '}') {
braceCount--;
//System.out.println("Brace count is "+braceCount+".");
}
if (allowed) {
sb.append((char) cp);
}
}
sb.append("}");
//System.out.println("=============");
//System.out.println(sb.toString());
return sb.toString();
}
public static JSONObject readJsonFromUrlWithFilter(String url, HashMap<Long,String> filter) throws IOException, JSONException {
return readJsonFromUrlWithFilter(url,filter,null,false);
}
public static JSONObject readJsonFromFileWithFilter(String file, HashMap<Long,String> filter) throws IOException, JSONException {
InputStream is = new FileInputStream(new File(file));
try {
BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
String jsonText = readFilter(rd,filter);
JSONObject json = new JSONObject(jsonText);
jsonText=null;
return json;
} finally {
is.close();
}
}
public static JSONObject readJsonFromUrlWithFilter(String url, HashMap<Long,String> filter, String file, boolean writeToFile) throws IOException, JSONException {
InputStream is = new URL(url).openStream();
try {
BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
String jsonText = readFilter(rd,filter);
if (writeToFile) {
writetoFile(new String[]{jsonText},file);
}
JSONObject json = new JSONObject(jsonText);
return json;
} finally {
is.close();
}
}
public static JSONObject readJsonFromUrl(String url) throws IOException, JSONException {
return readJsonFromUrl(url,null,false);
}
public static JSONArray readJsonArrayFromUrl(String url) throws IOException, JSONException {
return readJsonArrayFromUrl(url,null,false);
}
public static JSONObject readJsonFromFile(String file) throws IOException, JSONException {
InputStream is = new FileInputStream(new File(file));
try {
BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
String jsonText = readAll(rd);
JSONObject json = new JSONObject(jsonText);
jsonText=null;
return json;
} finally {
is.close();
}
}
public static JSONObject readJsonFromFile(File file) throws IOException, JSONException {
InputStream is = new FileInputStream(file);
try {
BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
String jsonText = readAll(rd);
JSONObject json = new JSONObject(jsonText);
jsonText=null;
return json;
} finally {
is.close();
}
}
public static JSONObject readJsonFromUrl(String url, String file, boolean writeToFile) throws IOException, JSONException {
//InputStream is = new URL(url).openStream();
URL site = new URL(url);
HttpURLConnection connection = (HttpURLConnection) site.openConnection();
/*for (String s : connection.getHeaderFields().keySet()) {
System.out.println(s+": "+connection.getHeaderFields().get(s));
}*/
//connection.setRequestMethod("GET");
connection.setRequestProperty("Content-Type", "application/json");
connection.setRequestProperty("User-Agent", "Mozilla/5.0");
int response = connection.getResponseCode();
//System.out.println("Response: "+response);
InputStreamReader stream = new InputStreamReader(connection.getInputStream());
BufferedReader rd = new BufferedReader(stream);
String jsonText = readAll(rd);
if (writeToFile) {
writetoFile(new String[]{jsonText},file);
}
JSONObject json = new JSONObject(jsonText);
stream.close();
return json;
}
static int LastSlash(String s) {
int lastSlashpos = 0;
for (int i=0;i<s.length();i++) {
if (s.charAt(i)=='/') {
lastSlashpos=i;
}
}
return lastSlashpos;
}
public static void downloadFileFromUrl(String url, String file) throws IOException, JSONException {
String temp = url.substring(0,LastSlash(url));
String temp2 = url.substring(LastSlash(url));
URL website = new URL(url);
HttpURLConnection connection = (HttpURLConnection) website.openConnection();
/*for (String s : connection.getHeaderFields().keySet()) {
System.out.println(s+": "+connection.getHeaderFields().get(s));
}*/
//connection.setRequestMethod("GET");
connection.setRequestProperty("Content-Type", "application/json");
connection.setRequestProperty("User-Agent", "Mozilla/5.0");
ReadableByteChannel rbc = Channels.newChannel(connection.getInputStream());
FileOutputStream fos = new FileOutputStream(file);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
}
public static JSONArray readJsonArrayFromUrl(String url, String file, boolean writeToFile) throws IOException, JSONException {
//InputStream is = new URL(url).openStream();
URL site = new URL(url);
HttpURLConnection connection = (HttpURLConnection) site.openConnection();
/*for (String s : connection.getHeaderFields().keySet()) {
System.out.println(s+": "+connection.getHeaderFields().get(s));
}*/
//connection.setRequestMethod("GET");
connection.setRequestProperty("Content-Type", "application/json");
connection.setRequestProperty("User-Agent", "Mozilla/5.0");
int response = connection.getResponseCode();
//System.out.println("Response: "+response);
InputStreamReader stream = new InputStreamReader(connection.getInputStream());
BufferedReader rd = new BufferedReader(stream);
String jsonText = readAll(rd);
if (writeToFile) {
writetoFile(new String[]{jsonText},file);
}
JSONArray json = new JSONArray(jsonText);
stream.close();
return json;
}
/*public static void logToFile(String message, String filename) {
logToFile(message,filename,false);
}*/
/*public static void logToFile(String message, String filename, boolean outputToChatLog) {
File file = new File(filename);
try {
if (!file.exists()) {
file.createNewFile();
}
FileWriter fw = new FileWriter(file, true);
PrintWriter pw = new PrintWriter(fw);
pw.println(message);
pw.flush();
pw.close();
} catch (IOException e) {
e.printStackTrace();
}
if (outputToChatLog && sigIRC.chatlogmodule_enabled) {
ChatLogMessage.importMessages(message);
}
}*/
public static void writetoFile(String[] data, String filename) {
File file = new File(filename);
try {
if (!file.exists()) {
file.createNewFile();
}
FileWriter fw = new FileWriter(file,false);
PrintWriter pw = new PrintWriter(fw);
for (String s : data) {
pw.println(s);
}
pw.flush();
pw.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public static void copyFile(File source, File dest) throws IOException {
FileChannel sourceChannel = null;
FileChannel destChannel = null;
try {
sourceChannel = new FileInputStream(source).getChannel();
destChannel = new FileOutputStream(dest).getChannel();
destChannel.transferFrom(sourceChannel, 0, sourceChannel.size());
}finally{
sourceChannel.close();
destChannel.close();
}
}
public static void deleteFile(String filename) {
File file = new File(filename);
if (file.exists()) {
file.delete();
}
}
}
Write Preview
Loading…
Cancel
Save