commit
1affa36254
@ -0,0 +1,13 @@ |
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?> |
||||
<launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType"> |
||||
<booleanAttribute key="org.eclipse.ant.ui.ATTR_TARGETS_UPDATED" value="true"/> |
||||
<booleanAttribute key="org.eclipse.ant.ui.DEFAULT_VM_INSTALL" value="false"/> |
||||
<booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="false"/> |
||||
<stringAttribute key="org.eclipse.jdt.launching.CLASSPATH_PROVIDER" value="org.eclipse.ant.ui.AntClasspathProvider"/> |
||||
<booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="true"/> |
||||
<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="Tagger"/> |
||||
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${workspace_loc:/Tagger/src/projectBuilder.xml}"/> |
||||
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS" value="full,incremental,"/> |
||||
<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/> |
||||
<stringAttribute key="org.eclipse.ui.externaltools.ATTR_WORKING_DIRECTORY" value="${workspace_loc:/Tagger}"/> |
||||
</launchConfiguration> |
@ -0,0 +1,27 @@ |
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<projectDescription> |
||||
<name>Tagger</name> |
||||
<comment></comment> |
||||
<projects> |
||||
</projects> |
||||
<buildSpec> |
||||
<buildCommand> |
||||
<name>org.eclipse.jdt.core.javabuilder</name> |
||||
<arguments> |
||||
</arguments> |
||||
</buildCommand> |
||||
<buildCommand> |
||||
<name>org.eclipse.ui.externaltools.ExternalToolBuilder</name> |
||||
<triggers>full,incremental,</triggers> |
||||
<arguments> |
||||
<dictionary> |
||||
<key>LaunchConfigHandle</key> |
||||
<value><project>/.externalToolBuilders/New_Builder.launch</value> |
||||
</dictionary> |
||||
</arguments> |
||||
</buildCommand> |
||||
</buildSpec> |
||||
<natures> |
||||
<nature>org.eclipse.jdt.core.javanature</nature> |
||||
</natures> |
||||
</projectDescription> |
Binary file not shown.
@ -0,0 +1,7 @@ |
||||
/temp10131499.html |
||||
/temp48528896.html |
||||
/temp51582658.html |
||||
/temp68277325.html |
||||
/temp76863890.html |
||||
/temp76890277.html |
||||
/temp76928198.html |
@ -0,0 +1,8 @@ |
||||
jpg |
||||
jpeg |
||||
tga |
||||
tif |
||||
tiff |
||||
png |
||||
bmp |
||||
gif |
@ -0,0 +1 @@ |
||||
/exiftool.exe |
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,35 @@ |
||||
import java.io.File; |
||||
|
||||
import javax.swing.JFileChooser; |
||||
|
||||
public class FileChooser { |
||||
|
||||
public FileChooser() { |
||||
JFileChooser chooser = new JFileChooser(); |
||||
chooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); |
||||
int val = chooser.showOpenDialog(null); |
||||
|
||||
FilterFiles filter = new FilterFiles(); |
||||
|
||||
if (val==JFileChooser.APPROVE_OPTION) { |
||||
File[] files = chooser.getSelectedFile().listFiles(filter); |
||||
for (File file : files) { |
||||
if (file.isFile()) { |
||||
String filename = file.getName(); |
||||
String newName = ReplaceExtraBits(filename); |
||||
System.out.println("Adding to Pixiv Album: " + newName); |
||||
imageTag.pixiv_image_list.add(newName); |
||||
imageTag.pixiv_rawimage_list.add(file); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
String ReplaceExtraBits(String filename) { |
||||
return filename |
||||
.replaceAll("\\..{1,}", "") //regex to replace file extensions
|
||||
.replaceAll("\\(.{1,}\\)", "") //regex to remove weird trimead parenthesis
|
||||
.replaceAll("_p.{1,}","") //Remove the p stuff from pixiv images.
|
||||
.trim(); //trim whitespace
|
||||
} |
||||
} |
@ -0,0 +1,26 @@ |
||||
import java.io.BufferedReader; |
||||
import java.io.BufferedWriter; |
||||
import java.io.File; |
||||
import java.io.FileNotFoundException; |
||||
import java.io.FileReader; |
||||
import java.io.FileWriter; |
||||
import java.io.FilenameFilter; |
||||
import java.io.IOException; |
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
public class FilterFiles implements FilenameFilter{ |
||||
public boolean accept(File f, String path) { |
||||
if (!path.matches(".{1,}_p[0-9]{1,}.{1,}")) { |
||||
System.out.println(path+" is not a pixiv image!"); |
||||
return false; |
||||
} |
||||
for (String filter : imageTag.filters.filters) { |
||||
if (path.toLowerCase().endsWith(filter.toLowerCase())) { |
||||
//System.out.println("Accepting "+path+" because of filter "+filter);
|
||||
return true; |
||||
} |
||||
} |
||||
return false; |
||||
} |
||||
} |
@ -0,0 +1,24 @@ |
||||
import java.io.BufferedReader; |
||||
import java.io.File; |
||||
import java.io.FileReader; |
||||
import java.io.IOException; |
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
public class Filters { |
||||
List<String> filters = new ArrayList<String>(); |
||||
public Filters() throws IOException { |
||||
File f = new File("filters.txt"); |
||||
FileReader fr = new FileReader(f); |
||||
BufferedReader br = new BufferedReader(fr); |
||||
String s = br.readLine(); |
||||
while (s!=null) { |
||||
String newfilter = s.trim(); |
||||
filters.add(newfilter); |
||||
System.out.println("Adding filter "+newfilter); |
||||
s = br.readLine(); |
||||
} |
||||
br.close(); |
||||
fr.close(); |
||||
} |
||||
} |
@ -0,0 +1,212 @@ |
||||
import java.io.BufferedWriter; |
||||
import java.io.File; |
||||
import java.io.FileWriter; |
||||
import java.io.IOException; |
||||
import java.net.URL; |
||||
import java.util.ArrayList; |
||||
import java.util.Arrays; |
||||
import java.util.Comparator; |
||||
import java.util.Iterator; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.Set; |
||||
import java.util.SortedSet; |
||||
import java.util.TreeMap; |
||||
import java.util.TreeSet; |
||||
|
||||
import org.apache.commons.imaging.ImageReadException; |
||||
import org.json.JSONArray; |
||||
import org.json.JSONObject; |
||||
|
||||
public class PixivManager { |
||||
public PixivManager() { |
||||
File folder = new File("downloadedData"); |
||||
if (folder.exists()) { |
||||
for (File fff : folder.listFiles()) { |
||||
if (fff.isFile()) { |
||||
fff.delete(); |
||||
} |
||||
} |
||||
} |
||||
folder.mkdirs(); |
||||
File outputTest = new File("TAG_DATA.txt"); |
||||
FileWriter fwOutput; |
||||
BufferedWriter bwOutput; |
||||
try { |
||||
fwOutput = new FileWriter(outputTest,true); |
||||
bwOutput = new BufferedWriter(fwOutput); |
||||
|
||||
for (String s : imageTag.pixiv_image_list) { |
||||
String url = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id="+s; |
||||
try { |
||||
if (!new File("downloadedData/temp"+s+".html").exists()) { |
||||
System.out.println("Starting download of "+url+" ..."); |
||||
utils.downloadFileFromUrl(url, "downloadedData/temp"+s+".html"); |
||||
String[] data = utils.readFromFile("downloadedData/temp"+s+".html"); |
||||
int scriptEndLine = 0; |
||||
while (scriptEndLine<data.length) { |
||||
if (data[scriptEndLine].contains("return Object.freeze(arg)")) { |
||||
break; |
||||
} |
||||
scriptEndLine++; |
||||
} |
||||
if (scriptEndLine==data.length) { |
||||
System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); |
||||
System.out.println(" IMAGE "+s+" FAILED TO PARSE CORRECTLY! Something is messed up about the file!!"); |
||||
System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); |
||||
} |
||||
File finaldata = new File("finaltemp"); |
||||
FileWriter fw; |
||||
try { |
||||
fw = new FileWriter(finaldata); |
||||
BufferedWriter bw = new BufferedWriter(fw); |
||||
int cutpos = data[scriptEndLine+1].indexOf("})(")+3; |
||||
if (cutpos<data[scriptEndLine+1].length()) { |
||||
bw.write(data[scriptEndLine+1].substring(cutpos,data[scriptEndLine+1].indexOf(");</script>"))); |
||||
} |
||||
bw.close(); |
||||
fw.close(); |
||||
} catch (IOException e) { |
||||
e.printStackTrace(); |
||||
} |
||||
JSONObject jsonData = utils.readJsonFromFile("finaltemp"); |
||||
//System.out.println(Arrays.deepToString(JSONObject.getNames(jsonData.getJSONObject("preload"))));
|
||||
//System.out.println(Arrays.deepToString(JSONObject.getNames(jsonData.getJSONObject("preload").getJSONObject("illust"))));
|
||||
JSONArray tagsArray = jsonData.getJSONObject("preload").getJSONObject("illust").getJSONObject(s).getJSONObject("tags").getJSONArray("tags"); |
||||
for (int i=0;i<tagsArray.length();i++) { |
||||
boolean hasEnglishTag=false; |
||||
JSONObject tag = tagsArray.getJSONObject(i); |
||||
String ENTag=""; |
||||
//String romaji="";
|
||||
/*if (tag.has("romaji") && !tag.isNull("romaji")) { |
||||
romaji = tag.getString("romaji"); |
||||
}*/ |
||||
if (tag.has("translation")) { |
||||
JSONObject translationObj = tag.getJSONObject("translation"); |
||||
if (translationObj.has("en")) { |
||||
hasEnglishTag=true; |
||||
ENTag = translationObj.getString("en"); |
||||
} |
||||
} else |
||||
if (tag.has("tag") && /*romaji.length()==0 &&*/ !tag.getString("tag").matches(".*\\p{InHiragana}.*")) { |
||||
hasEnglishTag=true; |
||||
ENTag = tag.getString("tag"); |
||||
} |
||||
|
||||
if (ENTag.replaceAll("\\?", "").trim().length()==0) { |
||||
ENTag=""; |
||||
hasEnglishTag=false; |
||||
} |
||||
boolean tagSubmitted=false; |
||||
String insertedTag=""; |
||||
if (hasEnglishTag && ENTag.length()>0) { |
||||
insertedTag = ENTag; |
||||
tagSubmitted=true; |
||||
} /*else |
||||
if (romaji.length()>0){ |
||||
insertedTag = romaji; |
||||
tagSubmitted=true; |
||||
}*/ |
||||
if (tagSubmitted) { |
||||
if (imageTag.tag_whitelist.size()==0 || imageTag.tag_whitelist.containsKey(insertedTag)) { |
||||
if (imageTag.taglist.containsKey(s)) { |
||||
List<String> tags = imageTag.taglist.get(s); |
||||
tags.add(insertedTag); |
||||
imageTag.taglist.put(s, tags); |
||||
} else { |
||||
List<String> tags = new ArrayList<String>(); |
||||
tags.add(insertedTag); |
||||
imageTag.taglist.put(s,tags); |
||||
} |
||||
if (imageTag.tagCounter.containsKey(insertedTag)) { |
||||
imageTag.tagCounter.put(insertedTag,imageTag.tagCounter.get(insertedTag)+1); |
||||
} else { |
||||
imageTag.tagCounter.put(insertedTag,1); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
String taglist = s+": <"+imageTag.taglist.get(s)+">"; |
||||
System.out.println(taglist); |
||||
bwOutput.append(taglist); |
||||
bwOutput.newLine(); |
||||
//jsonData.getJSONObject("preload").getJSONObject("illust").getJSONObject(s).getJSONObject("tags");
|
||||
} else { |
||||
System.out.println("Skipping image "+s+" because it has already been processed."); |
||||
} |
||||
} catch (IOException e) { |
||||
e.printStackTrace(); |
||||
} |
||||
/*org.apache.commons.io.FileUtils.copyURLToFile(new URL( |
||||
url |
||||
),temp);*/ |
||||
} |
||||
|
||||
/*for (String s : imageTag.taglist.keySet()) { |
||||
System.out.println(s+": <"+imageTag.taglist.get(s)+">"); |
||||
}*/ |
||||
|
||||
bwOutput.close(); |
||||
fwOutput.close(); |
||||
} catch (IOException e1) { |
||||
e1.printStackTrace(); |
||||
} |
||||
Map sorted = sortByValues(imageTag.tagCounter); |
||||
Set s = sorted.entrySet(); |
||||
Iterator i = s.iterator(); |
||||
File tagFile = new File("SORTED_TAGS.txt"); |
||||
FileWriter fw; |
||||
try { |
||||
fw = new FileWriter(tagFile); |
||||
BufferedWriter bw = new BufferedWriter(fw); |
||||
while (i.hasNext()) { |
||||
Map.Entry m = (Map.Entry)i.next(); |
||||
String key = (String)m.getKey(); |
||||
Integer value = (Integer)m.getValue(); |
||||
bw.write(key+" - "+value); |
||||
bw.newLine(); |
||||
} |
||||
bw.close(); |
||||
fw.close(); |
||||
} catch (IOException e1) { |
||||
e1.printStackTrace(); |
||||
} |
||||
|
||||
if (imageTag.tag_whitelist.size()==0) { |
||||
System.out.println("whitelist.txt not found! No tagging will be done this time. TAG_DATA.txt populated."); |
||||
} else { |
||||
System.out.println("Tagging Images..."); |
||||
for (int j=0;j<imageTag.pixiv_image_list.size();j++) { |
||||
try { |
||||
Tagger tags = new Tagger(imageTag.pixiv_rawimage_list.get(j),imageTag.taglist.get(imageTag.pixiv_image_list.get(j))); |
||||
} catch (ImageReadException e) { |
||||
e.printStackTrace(); |
||||
} catch (IOException e) { |
||||
e.printStackTrace(); |
||||
} |
||||
//System.out.println("Tagged "+ss+" with "+tagString);
|
||||
} |
||||
} |
||||
} |
||||
|
||||
|
||||
public static <K, V extends Comparable<V>> Map<K, V> |
||||
sortByValues(final Map<K, V> map) { |
||||
Comparator<K> valueComparator = |
||||
new Comparator<K>() { |
||||
public int compare(K k1, K k2) { |
||||
int compare = |
||||
-map.get(k1).compareTo(map.get(k2)); |
||||
if (compare == 0) |
||||
return 1; |
||||
else |
||||
return compare; |
||||
} |
||||
}; |
||||
|
||||
Map<K, V> sortedByValues = |
||||
new TreeMap<K, V>(valueComparator); |
||||
sortedByValues.putAll(map); |
||||
return sortedByValues; |
||||
} |
||||
} |
@ -0,0 +1,42 @@ |
||||
import java.io.BufferedReader; |
||||
import java.io.File; |
||||
import java.io.IOException; |
||||
import java.io.InputStream; |
||||
import java.io.InputStreamReader; |
||||
import java.util.List; |
||||
|
||||
import org.apache.commons.imaging.ImageReadException; |
||||
import org.apache.commons.imaging.ImageWriteException; |
||||
import org.apache.commons.imaging.Imaging; |
||||
import org.apache.commons.imaging.common.ImageMetadata; |
||||
import org.apache.commons.imaging.formats.jpeg.JpegImageMetadata; |
||||
import org.apache.commons.imaging.formats.tiff.TiffField; |
||||
import org.apache.commons.imaging.formats.tiff.TiffImageMetadata; |
||||
import org.apache.commons.imaging.formats.tiff.constants.ExifTagConstants; |
||||
import org.apache.commons.imaging.formats.tiff.constants.MicrosoftTagConstants; |
||||
import org.apache.commons.imaging.formats.tiff.write.TiffOutputDirectory; |
||||
import org.apache.commons.imaging.formats.tiff.write.TiffOutputField; |
||||
import org.apache.commons.imaging.formats.tiff.write.TiffOutputSet; |
||||
|
||||
public class Tagger { |
||||
public Tagger(File imagetoTag,List<String> tags) throws ImageReadException, IOException { |
||||
StringBuilder sb = new StringBuilder(); |
||||
if (tags!=null) { |
||||
for (String tag : tags) { |
||||
if (sb.length()!=0) { |
||||
sb.append("; "); |
||||
} |
||||
sb.append(tag); |
||||
} |
||||
|
||||
Process tool = Runtime.getRuntime().exec("tool.exe -exif:XPKeywords=\""+sb.toString()+"\" "+imagetoTag.getAbsolutePath()+" -overwrite_original_in_place -P"); |
||||
|
||||
BufferedReader stdInput = new BufferedReader(new |
||||
InputStreamReader(tool.getInputStream())); |
||||
String s = null; |
||||
while ((s = stdInput.readLine()) != null) { |
||||
System.out.println("Tagging "+imagetoTag.getName()+":"+s); |
||||
} |
||||
} |
||||
} |
||||
} |
Binary file not shown.
@ -0,0 +1,91 @@ |
||||
import java.io.BufferedReader; |
||||
import java.io.File; |
||||
import java.io.FileFilter; |
||||
import java.io.FileNotFoundException; |
||||
import java.io.FileOutputStream; |
||||
import java.io.FileReader; |
||||
import java.io.FileWriter; |
||||
import java.io.FilenameFilter; |
||||
import java.io.IOException; |
||||
import java.io.InputStream; |
||||
import java.io.OutputStream; |
||||
import java.util.ArrayList; |
||||
import java.util.HashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.SortedMap; |
||||
import java.util.TreeMap; |
||||
|
||||
import javax.swing.JFileChooser; |
||||
|
||||
public class imageTag { |
||||
public static Filters filters; |
||||
public static HashMap<String,Boolean> tag_whitelist = new HashMap<String,Boolean>(); |
||||
public static List<String> pixiv_image_list = new ArrayList<String>(); |
||||
public static List<File> pixiv_rawimage_list = new ArrayList<File>(); |
||||
public static HashMap<String,List<String>> taglist = new HashMap<String,List<String>>(); |
||||
public static Map<String,Integer> tagCounter = new TreeMap<String,Integer>(); |
||||
|
||||
public static void main(String[] args) { |
||||
|
||||
ExtractExifTool(); |
||||
|
||||
try { |
||||
filters = new Filters(); |
||||
} catch (IOException e) { |
||||
e.printStackTrace(); |
||||
} |
||||
|
||||
GetTagWhitelist(); |
||||
|
||||
FileChooser fc = new FileChooser(); |
||||
|
||||
PixivManager pm = new PixivManager(); |
||||
|
||||
System.out.println("Done!"); |
||||
} |
||||
|
||||
private static void GetTagWhitelist() { |
||||
File whitelist = new File("whitelist.txt"); |
||||
if (whitelist.exists()) { |
||||
FileReader fr; |
||||
try { |
||||
fr = new FileReader(whitelist); |
||||
BufferedReader br = new BufferedReader(fr); |
||||
String s; |
||||
try { |
||||
s = br.readLine(); |
||||
while (s!=null) { |
||||
String newtag = s.trim(); |
||||
tag_whitelist.put(newtag,true); |
||||
System.out.println("Read in whitelisted tag: "+newtag); |
||||
s=br.readLine(); |
||||
} |
||||
} catch (IOException e) { |
||||
e.printStackTrace(); |
||||
} |
||||
} catch (FileNotFoundException e1) { |
||||
e1.printStackTrace(); |
||||
} |
||||
} |
||||
} |
||||
|
||||
private static void ExtractExifTool() { |
||||
InputStream tool = imageTag.class.getResourceAsStream("exiftool.exe"); |
||||
File f = new File("tool.exe"); |
||||
try { |
||||
byte[] buffer = new byte[tool.available()]; |
||||
tool.read(buffer); |
||||
OutputStream outStream = new FileOutputStream(f); |
||||
outStream.write(buffer); |
||||
outStream.close(); |
||||
tool.close(); |
||||
f.deleteOnExit(); |
||||
} catch (IOException e) { |
||||
e.printStackTrace(); |
||||
} |
||||
|
||||
} |
||||
} |
||||
|
||||
|
@ -0,0 +1,20 @@ |
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?> |
||||
<project default="create_run_jar" name="Create Runnable Jar for Project sigIRCv2"> |
||||
<!--this file was created by Eclipse Runnable JAR Export Wizard--> |
||||
<!--ANT 1.7 is required --> |
||||
<!--define folder properties--> |
||||
<property name="dir.buildfile" value="."/> |
||||
<property name="dir.workspace" value="${dir.buildfile}/.."/> |
||||
<property name="dir.jarfile" value="${dir.buildfile}"/> |
||||
<target name="create_run_jar"> |
||||
<jar destfile="${dir.jarfile}/Tagger.jar" filesetmanifest="mergewithoutmain"> |
||||
<manifest> |
||||
<attribute name="Main-Class" value="imageTag"/> |
||||
<attribute name="Class-Path" value="."/> |
||||
</manifest> |
||||
<fileset dir="${dir.jarfile}/bin"/> |
||||
<zipfileset excludes="META-INF/*.SF" src="lib/commons-imaging-1.0-alpha1.jar"/> |
||||
<zipfileset excludes="META-INF/*.SF" src="lib/json-20190722.jar"/> |
||||
</jar> |
||||
</target> |
||||
</project> |
@ -0,0 +1,365 @@ |
||||
import java.io.BufferedReader; |
||||
import java.io.File; |
||||
import java.io.FileInputStream; |
||||
import java.io.FileOutputStream; |
||||
import java.io.FileReader; |
||||
import java.io.FileWriter; |
||||
import java.io.IOException; |
||||
import java.io.InputStream; |
||||
import java.io.InputStreamReader; |
||||
import java.io.PrintWriter; |
||||
import java.io.Reader; |
||||
import java.io.UnsupportedEncodingException; |
||||
import java.net.HttpURLConnection; |
||||
import java.net.URL; |
||||
import java.net.URLConnection; |
||||
import java.net.URLEncoder; |
||||
import java.nio.ByteBuffer; |
||||
import java.nio.CharBuffer; |
||||
import java.nio.channels.Channels; |
||||
import java.nio.channels.FileChannel; |
||||
import java.nio.channels.ReadableByteChannel; |
||||
import java.nio.charset.Charset; |
||||
import java.util.ArrayList; |
||||
import java.util.HashMap; |
||||
import java.util.List; |
||||
|
||||
import org.json.JSONArray; |
||||
import org.json.JSONException; |
||||
import org.json.JSONObject; |
||||
public class utils { |
||||
public static String[] readFromFile(String filename) { |
||||
File file = new File(filename); |
||||
//System.out.println(file.getAbsolutePath());
|
||||
List<String> contents= new ArrayList<String>(); |
||||
if (file.exists()) { |
||||
try( |
||||
FileReader fw = new FileReader(filename); |
||||
BufferedReader bw = new BufferedReader(fw);) |
||||
{ |
||||
String readline = bw.readLine(); |
||||
do { |
||||
if (readline!=null) { |
||||
//System.out.println(readline);
|
||||
contents.add(readline); |
||||
readline = bw.readLine(); |
||||
}} while (readline!=null); |
||||
fw.close(); |
||||
bw.close(); |
||||
} catch (IOException e) { |
||||
e.printStackTrace(); |
||||
} |
||||
} |
||||
return contents.toArray(new String[contents.size()]); |
||||
} |
||||
|
||||
private static String readAll(Reader rd) throws IOException { |
||||
StringBuilder sb = new StringBuilder(); |
||||
int cp; |
||||
while ((cp = rd.read()) != -1) { |
||||
sb.append((char) cp); |
||||
} |
||||
return sb.toString(); |
||||
} |
||||
|
||||
private static String readFilter(Reader rd, HashMap<Long,String> channel_ids) throws IOException { |
||||
StringBuilder sb = new StringBuilder(); |
||||
boolean allowed=false; |
||||
boolean quotation_mark=false; |
||||
boolean endquotation_mark=false; |
||||
boolean foundChannel=false; |
||||
boolean nextBrace=false; |
||||
boolean outputStuff=false; |
||||
String numb = ""; |
||||
int braceCount=0; |
||||
int channelCount=0; |
||||
int vals=0; |
||||
int cp; |
||||
while ((cp = rd.read()) != -1) { |
||||
if (braceCount==0) { |
||||
allowed=true; |
||||
} else |
||||
if (braceCount==1 && !quotation_mark){ |
||||
quotation_mark=true; |
||||
numb=""; |
||||
allowed=false; |
||||
} else |
||||
if (!endquotation_mark) { |
||||
if ((char)cp >= '0' && |
||||
(char)cp <= '9') { |
||||
allowed=false; |
||||
numb+=(char)cp; |
||||
} else { |
||||
allowed=false; |
||||
endquotation_mark=true; |
||||
try { |
||||
if (channel_ids.containsKey(Long.parseLong(numb))) { |
||||
if (channelCount>=1) { |
||||
sb.append(","); |
||||
} |
||||
sb.append("\""+numb+"\""); |
||||
foundChannel=true; |
||||
System.out.println("Found channel "+numb); |
||||
outputStuff=true; |
||||
} |
||||
} catch (NumberFormatException e) { |
||||
|
||||
} |
||||
} |
||||
} else |
||||
if (!nextBrace && foundChannel) { |
||||
allowed=true; |
||||
if ((char)cp == '{') { |
||||
nextBrace=true; |
||||
} |
||||
} else |
||||
if (foundChannel) { |
||||
allowed=true; |
||||
if (braceCount==1) { |
||||
allowed=false; |
||||
channelCount++; |
||||
quotation_mark=false; |
||||
endquotation_mark=false; |
||||
foundChannel=false; |
||||
nextBrace=false; |
||||
} |
||||
} else { |
||||
allowed=false; |
||||
if (braceCount==1) { |
||||
allowed=false; |
||||
quotation_mark=false; |
||||
endquotation_mark=false; |
||||
foundChannel=false; |
||||
nextBrace=false; |
||||
} |
||||
} |
||||
|
||||
/*if (outputStuff && vals++<1000) { |
||||
System.out.print((char)cp); |
||||
}*/ |
||||
if ((char)cp == '{') { |
||||
braceCount++; |
||||
//System.out.println("Brace count is "+braceCount+".");
|
||||
} else |
||||
if ((char)cp == '}') { |
||||
braceCount--; |
||||
//System.out.println("Brace count is "+braceCount+".");
|
||||
} |
||||
|
||||
if (allowed) { |
||||
sb.append((char) cp); |
||||
} |
||||
} |
||||
sb.append("}"); |
||||
//System.out.println("=============");
|
||||
//System.out.println(sb.toString());
|
||||
return sb.toString(); |
||||
} |
||||
|
||||
|
||||
public static JSONObject readJsonFromUrlWithFilter(String url, HashMap<Long,String> filter) throws IOException, JSONException { |
||||
return readJsonFromUrlWithFilter(url,filter,null,false); |
||||
} |
||||
|
||||
public static JSONObject readJsonFromFileWithFilter(String file, HashMap<Long,String> filter) throws IOException, JSONException { |
||||
InputStream is = new FileInputStream(new File(file)); |
||||
try { |
||||
BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8"))); |
||||
String jsonText = readFilter(rd,filter); |
||||
JSONObject json = new JSONObject(jsonText); |
||||
jsonText=null; |
||||
return json; |
||||
} finally { |
||||
is.close(); |
||||
} |
||||
} |
||||
|
||||
public static JSONObject readJsonFromUrlWithFilter(String url, HashMap<Long,String> filter, String file, boolean writeToFile) throws IOException, JSONException { |
||||
InputStream is = new URL(url).openStream(); |
||||
try { |
||||
BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8"))); |
||||
String jsonText = readFilter(rd,filter); |
||||
if (writeToFile) { |
||||
writetoFile(new String[]{jsonText},file); |
||||
} |
||||
JSONObject json = new JSONObject(jsonText); |
||||
return json; |
||||
} finally { |
||||
is.close(); |
||||
} |
||||
} |
||||
|
||||
public static JSONObject readJsonFromUrl(String url) throws IOException, JSONException { |
||||
return readJsonFromUrl(url,null,false); |
||||
} |
||||
|
||||
public static JSONArray readJsonArrayFromUrl(String url) throws IOException, JSONException { |
||||
return readJsonArrayFromUrl(url,null,false); |
||||
} |
||||
|
||||
public static JSONObject readJsonFromFile(String file) throws IOException, JSONException { |
||||
InputStream is = new FileInputStream(new File(file)); |
||||
try { |
||||
BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8"))); |
||||
String jsonText = readAll(rd); |
||||
JSONObject json = new JSONObject(jsonText); |
||||
jsonText=null; |
||||
return json; |
||||
} finally { |
||||
is.close(); |
||||
} |
||||
} |
||||
public static JSONObject readJsonFromFile(File file) throws IOException, JSONException { |
||||
InputStream is = new FileInputStream(file); |
||||
try { |
||||
BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8"))); |
||||
String jsonText = readAll(rd); |
||||
JSONObject json = new JSONObject(jsonText); |
||||
jsonText=null; |
||||
return json; |
||||
} finally { |
||||
is.close(); |
||||
} |
||||
} |
||||
|
||||
public static JSONObject readJsonFromUrl(String url, String file, boolean writeToFile) throws IOException, JSONException { |
||||
//InputStream is = new URL(url).openStream();
|
||||
URL site = new URL(url); |
||||
HttpURLConnection connection = (HttpURLConnection) site.openConnection(); |
||||
/*for (String s : connection.getHeaderFields().keySet()) { |
||||
System.out.println(s+": "+connection.getHeaderFields().get(s)); |
||||
}*/ |
||||
//connection.setRequestMethod("GET");
|
||||
connection.setRequestProperty("Content-Type", "application/json"); |
||||
connection.setRequestProperty("User-Agent", "Mozilla/5.0"); |
||||
int response = connection.getResponseCode(); |
||||
//System.out.println("Response: "+response);
|
||||
InputStreamReader stream = new InputStreamReader(connection.getInputStream()); |
||||
BufferedReader rd = new BufferedReader(stream); |
||||
String jsonText = readAll(rd); |
||||
if (writeToFile) { |
||||
writetoFile(new String[]{jsonText},file); |
||||
} |
||||
JSONObject json = new JSONObject(jsonText); |
||||
stream.close(); |
||||
return json; |
||||
} |
||||
|
||||
static int LastSlash(String s) { |
||||
int lastSlashpos = 0; |
||||
for (int i=0;i<s.length();i++) { |
||||
if (s.charAt(i)=='/') { |
||||
lastSlashpos=i; |
||||
} |
||||
} |
||||
return lastSlashpos; |
||||
} |
||||
|
||||
public static void downloadFileFromUrl(String url, String file) throws IOException, JSONException { |
||||
String temp = url.substring(0,LastSlash(url)); |
||||
String temp2 = url.substring(LastSlash(url)); |
||||
|
||||
URL website = new URL(url); |
||||
HttpURLConnection connection = (HttpURLConnection) website.openConnection(); |
||||
/*for (String s : connection.getHeaderFields().keySet()) { |
||||
System.out.println(s+": "+connection.getHeaderFields().get(s)); |
||||
}*/ |
||||
//connection.setRequestMethod("GET");
|
||||
connection.setRequestProperty("Content-Type", "application/json"); |
||||
connection.setRequestProperty("User-Agent", "Mozilla/5.0"); |
||||
ReadableByteChannel rbc = Channels.newChannel(connection.getInputStream()); |
||||
FileOutputStream fos = new FileOutputStream(file); |
||||
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); |
||||
fos.close(); |
||||
} |
||||
|
||||
public static JSONArray readJsonArrayFromUrl(String url, String file, boolean writeToFile) throws IOException, JSONException { |
||||
//InputStream is = new URL(url).openStream();
|
||||
URL site = new URL(url); |
||||
HttpURLConnection connection = (HttpURLConnection) site.openConnection(); |
||||
/*for (String s : connection.getHeaderFields().keySet()) { |
||||
System.out.println(s+": "+connection.getHeaderFields().get(s)); |
||||
}*/ |
||||
//connection.setRequestMethod("GET");
|
||||
connection.setRequestProperty("Content-Type", "application/json"); |
||||
connection.setRequestProperty("User-Agent", "Mozilla/5.0"); |
||||
int response = connection.getResponseCode(); |
||||
//System.out.println("Response: "+response);
|
||||
InputStreamReader stream = new InputStreamReader(connection.getInputStream()); |
||||
BufferedReader rd = new BufferedReader(stream); |
||||
String jsonText = readAll(rd); |
||||
if (writeToFile) { |
||||
writetoFile(new String[]{jsonText},file); |
||||
} |
||||
JSONArray json = new JSONArray(jsonText); |
||||
stream.close(); |
||||
return json; |
||||
} |
||||
|
||||
/*public static void logToFile(String message, String filename) { |
||||
logToFile(message,filename,false); |
||||
}*/ |
||||
|
||||
/*public static void logToFile(String message, String filename, boolean outputToChatLog) { |
||||
File file = new File(filename); |
||||
try { |
||||
|
||||
if (!file.exists()) { |
||||
file.createNewFile(); |
||||
} |
||||
|
||||
FileWriter fw = new FileWriter(file, true); |
||||
PrintWriter pw = new PrintWriter(fw); |
||||
|
||||
pw.println(message); |
||||
pw.flush(); |
||||
pw.close(); |
||||
} catch (IOException e) { |
||||
e.printStackTrace(); |
||||
} |
||||
if (outputToChatLog && sigIRC.chatlogmodule_enabled) { |
||||
ChatLogMessage.importMessages(message); |
||||
} |
||||
}*/ |
||||
|
||||
public static void writetoFile(String[] data, String filename) { |
||||
File file = new File(filename); |
||||
try { |
||||
|
||||
if (!file.exists()) { |
||||
file.createNewFile(); |
||||
} |
||||
|
||||
FileWriter fw = new FileWriter(file,false); |
||||
PrintWriter pw = new PrintWriter(fw); |
||||
|
||||
for (String s : data) { |
||||
pw.println(s); |
||||
} |
||||
pw.flush(); |
||||
pw.close(); |
||||
} catch (IOException e) { |
||||
e.printStackTrace(); |
||||
} |
||||
} |
||||
|
||||
public static void copyFile(File source, File dest) throws IOException { |
||||
FileChannel sourceChannel = null; |
||||
FileChannel destChannel = null; |
||||
try { |
||||
sourceChannel = new FileInputStream(source).getChannel(); |
||||
destChannel = new FileOutputStream(dest).getChannel(); |
||||
destChannel.transferFrom(sourceChannel, 0, sourceChannel.size()); |
||||
}finally{ |
||||
sourceChannel.close(); |
||||
destChannel.close(); |
||||
} |
||||
} |
||||
|
||||
public static void deleteFile(String filename) { |
||||
File file = new File(filename); |
||||
if (file.exists()) { |
||||
file.delete(); |
||||
} |
||||
} |
||||
} |
Loading…
Reference in new issue