Improved memory footprint for retrieving sub emoticons.

src/sig/UpdateEvent.java

@@ -2,6 +2,14 @@ package sig;
 import java.awt.Component;
 import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
+import java.io.File;
+import java.io.IOException;
+import java.util.Calendar;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import sig.utils.FileUtils;
 
 public class UpdateEvent implements ActionListener{
 	final static int MSGTIMER = 300;
@@ -19,9 +27,59 @@ public class UpdateEvent implements ActionListener{
 		    UpdateScrollingText();
 		    UpdateAuthenticationCountdownMessage();
 		    UpdateWindowPosition();
+		    UpdateSubEmoticons();
 		}
 	}
 
+	private void UpdateSubEmoticons() {
+		if (!sigIRC.downloadedSubEmotes &&
+				sigIRC.subchannelCount==sigIRC.subchannelIds.size()) {
+			Thread downloadThread = new Thread(){
+				public void run() {
+						JSONObject data = GetSubEmoteJson();
+						sigIRC.downloadSubEmotes(data);
+					}
+				};
+			downloadThread.start();
+			sigIRC.downloadedSubEmotes=true;
+		}
+	}
+
+	private JSONObject GetSubEmoteJson() {
+		JSONObject subemotes = null;
+		try {
+			File filer = new File(sigIRC.SUBEMOTELISTFILE);
+			if (!filer.exists()) {
+				System.out.println("Local copy of Sub emotes not found. Downloading in background...");
+				subemotes = FileUtils.readJsonFromUrlWithFilter("https://twitchemotes.com/api_cache/v3/subscriber.json",sigIRC.subchannelIds,sigIRC.SUBEMOTELISTFILE,true);
+			} else {
+				if (sigIRC.lastSubEmoteUpdate == Calendar.getInstance().get(Calendar.DAY_OF_YEAR)) {
+					System.out.println("Using local copy of Sub emote JSON.");
+					subemotes = FileUtils.readJsonFromFileWithFilter(sigIRC.SUBEMOTELISTFILE,sigIRC.subchannelIds);
+				} else {
+					System.out.println("Local copy of Sub emote JSON out-of-date! Re-downloading in background...");
+					subemotes = FileUtils.readJsonFromFileWithFilter(sigIRC.SUBEMOTELISTFILE,sigIRC.subchannelIds);
+					new Thread(){
+						public void run() {
+							try {
+								FileUtils.readJsonFromUrlWithFilter("https://twitchemotes.com/api_cache/v3/subscriber.json",sigIRC.subchannelIds,sigIRC.SUBEMOTELISTFILE,true);
+							} catch (JSONException e) {
+								e.printStackTrace();
+							} catch (IOException e) {
+								e.printStackTrace();
+							}
+						}
+					}.start();
+				}
+			}
+		} catch (JSONException | IOException e) {
+			e.printStackTrace();
+		}
+		sigIRC.lastSubEmoteUpdate = Calendar.getInstance().get(Calendar.DAY_OF_YEAR);
+		sigIRC.config.setInteger("lastSubEmote_APIUpdate", sigIRC.lastSubEmoteUpdate);
+		return subemotes;
+	}
+
 	private void UpdateWindowPosition() {
 		if (windowUpdateCounter--<=0) {
 			if (sigIRC.lastWindowX!=(int)sigIRC.window.getLocationOnScreen().getX() ||

src/sig/sigIRC.java

@@ -42,6 +42,7 @@ import java.net.URL;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Calendar;
+import java.util.HashMap;
 import java.util.List;
 
 import javax.imageio.ImageIO;
@@ -121,6 +122,11 @@ public class sigIRC{
 	public static long channel_id = -1;
 	public static int lastSubEmoteUpdate = -1;
 	
+	public static int subchannelCount = 0;
+	public static HashMap<Long,String> subchannelIds = new HashMap<Long,String>();
+	
+	public static boolean downloadedSubEmotes=false;
+	
 	static int lastWindowX = 0;
 	static int lastWindowY = 0;
 
@@ -310,9 +316,8 @@ public class sigIRC{
 		}
 	}
 
-	private static void downloadSubEmotes(JSONObject emotes, String channelName) {
+	private static void getSubChannels(String channelName) {
 		manager.channels().get(channelName, new ChannelResponseHandler() {
-
 			@Override
 			public void onFailure(Throwable arg0) {
 			}
@@ -323,23 +328,8 @@ public class sigIRC{
 
 			@Override
 			public void onSuccess(Channel arg0) {
-				if (arg0!=null) {
+				subchannelIds.put(arg0.getId(),channelName);
-					channel_id=arg0.getId();
+				//System.out.println("Got ID "+arg0.getId()+" for channel "+channelName);
-					JSONObject channel = emotes.getJSONObject(Long.toString(channel_id));
-					JSONArray arr = channel.getJSONArray("emotes");
-					//System.out.println("Channel: "+channel);
-					for (int i=0;i<arr.length();i++) {
-						JSONObject emote = arr.getJSONObject(i);
-						int id = emote.getInt("id");
-						String name = emote.getString("code");
-						//System.out.println("Emote "+(i+1)+" has id "+id+" and code "+name+".");
-						try {
-							emoticon_queue.add(new SubEmoticon(name, new URL(TWITCHEMOTEURL+id+"/1.0"), channelName));
-						} catch (MalformedURLException e) {
-							e.printStackTrace();
-						}
-					}
-				}
 			}
 
 		});
@@ -347,6 +337,27 @@ public class sigIRC{
 		//return true;
 	}
 
+	public static void downloadSubEmotes(JSONObject data) {
+		for (Long l : subchannelIds.keySet()) {
+			JSONObject channel = data.getJSONObject(Long.toString(l));
+			JSONArray arr = channel.getJSONArray("emotes");
+			//System.out.println("Channel: "+channel);
+			for (int i=0;i<arr.length();i++) {
+				JSONObject emote = arr.getJSONObject(i);
+				int id = emote.getInt("id");
+				String name = emote.getString("code");
+				//System.out.println("Emote "+(i+1)+" has id "+id+" and code "+name+".");
+				try {
+					emoticon_queue.add(new SubEmoticon(name, new URL(TWITCHEMOTEURL+id+"/1.0"), subchannelIds.get(l)));
+				} catch (MalformedURLException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+		//TwitchModule.streamOnline=true;
+		//return true;
+	}
+
 	private static void performTwitchEmoteUpdate() {
 		try {
 			JSONObject twitchemotes = FileUtils.readJsonFromUrl("https://twitchemotes.com/api_cache/v3/global.json");
@@ -358,53 +369,17 @@ public class sigIRC{
 				emoticons.add(new Emoticon(name, new URL(TWITCHEMOTEURL+id+"/1.0")));
 				System.out.println("Emote "+id+" with name "+name);
 			}
-			Thread downloadThread = new Thread(){
-				public void run() {
-					JSONObject subemotes = null;
-					try {
-						File filer = new File(SUBEMOTELISTFILE);
-						if (!filer.exists()) {
-							System.out.println("Local copy of Sub emotes not found. Downloading in background...");
-							subemotes = FileUtils.readJsonFromUrl("https://twitchemotes.com/api_cache/v3/subscriber.json",SUBEMOTELISTFILE,true);
-						} else {
-							if (lastSubEmoteUpdate == Calendar.getInstance().get(Calendar.DAY_OF_YEAR)) {
-								System.out.println("Using local copy of Sub emote JSON.");
-								subemotes = FileUtils.readJsonFromFile(SUBEMOTELISTFILE);
-							} else {
-								System.out.println("Local copy of Sub emote JSON out-of-date! Re-downloading in background...");
-								subemotes = FileUtils.readJsonFromFile(SUBEMOTELISTFILE);
-								new Thread(){
-									public void run() {
-										try {
-											FileUtils.readJsonFromUrl("https://twitchemotes.com/api_cache/v3/subscriber.json",SUBEMOTELISTFILE,true);
-										} catch (JSONException e) {
-											e.printStackTrace();
-										} catch (IOException e) {
-											e.printStackTrace();
-										}
-									}
-								}.start();
-							}
-						}
-					} catch (JSONException | IOException e) {
-						e.printStackTrace();
-					}
-					lastSubEmoteUpdate = Calendar.getInstance().get(Calendar.DAY_OF_YEAR);
-					config.setInteger("lastSubEmote_APIUpdate", lastSubEmoteUpdate);
 					//System.out.println("Subscriber object: "+subemotes);
-					String[] sub_emotes = FileUtils.readFromFile(sigIRC.BASEDIR+"sigIRC/Emotes/subscribers.txt");
+					String[] channel_names = FileUtils.readFromFile(sigIRC.BASEDIR+"sigIRC/Emotes/subscribers.txt");
-					//System.out.println("Sub emotes read.");
+					subchannelCount = channel_names.length;
-					for (String s : sub_emotes) {
+					//System.out.println("Expecting "+subchannelCount+" Channel IDs.");
+					for (String s : channel_names) {
 						if (s.length()>0) {
 							s=s.trim();
-							System.out.println("Got sub emote info for "+s);
+							//System.out.println("Got sub emote info for "+s);
-							downloadSubEmotes(subemotes,s);
+							getSubChannels(s);
 						}
 					}
-					subemotes=null;
-				}
-			};
-			downloadThread.start();
 			/*JSONObject emotelist = twitchemotes.getJSONObject("emotes");
 			JSONObject templatelist = twitchemotes.getJSONObject("template");
 			String templateurl = templatelist.getString("small");

src/sig/utils/FileUtils.java

@@ -14,6 +14,7 @@ import java.net.URL;
 import java.nio.channels.FileChannel;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 
 import org.json.JSONException;
@@ -55,12 +56,110 @@ public class FileUtils {
 	    }
 	    return sb.toString();
 	  }
+	
+	  private static String readFilter(Reader rd, HashMap<Long,String> channel_ids) throws IOException {
+	    StringBuilder sb = new StringBuilder();
+	    boolean allowed=false;
+	    boolean quotation_mark=false;
+	    boolean endquotation_mark=false;
+	    boolean foundChannel=false;
+	    boolean nextBrace=false;
+	    boolean outputStuff=false;
+	    String numb = "";
+	    int braceCount=0;
+	    int channelCount=0;
+	    int vals=0;
+	    int cp;
+	    while ((cp = rd.read()) != -1) {
+	    	if (braceCount==0) {
+	    		allowed=true;
+	    	} else 
+	    	if (braceCount==1 && !quotation_mark){
+	    		quotation_mark=true;
+	    		numb="";
+	    		allowed=false;
+	    	} else
+	    	if (!endquotation_mark) {
+	    	  if ((char)cp >= '0' &&
+	    			  (char)cp <= '9') {
+	    		  allowed=false;
+	    		  numb+=(char)cp;
+	    	  } else {
+	    		  allowed=false;
+	    		  endquotation_mark=true;
+	    		  try {
+		    		  if (channel_ids.containsKey(Long.parseLong(numb))) {
+		    			  if (channelCount>=1) {
+		    				  sb.append(",");
+		    			  }
+		    			  sb.append("\""+numb+"\"");
+		    			  foundChannel=true;
+		    			  System.out.println("Found channel "+numb);
+		    			  outputStuff=true;
+		    		  }
+	    		  } catch (NumberFormatException e) {
+	    			  
+	    		  }
+	    	  }
+	    	} else 
+	    	if (!nextBrace && foundChannel) {
+	    		allowed=true;
+	    		if ((char)cp == '{') {
+	    			nextBrace=true;
+	    		}
+	    	} else
+	    	if (foundChannel) {
+	    		allowed=true;
+	    		if (braceCount==1) {
+	    			allowed=false;
+	    			channelCount++;
+	    			quotation_mark=false;
+	    			endquotation_mark=false;
+	    			foundChannel=false;
+	    			nextBrace=false;
+	    		}
+	    	} else {
+	    		allowed=false;
+	    		if (braceCount==1) {
+	    			allowed=false;
+	    			quotation_mark=false;
+	    			endquotation_mark=false;
+	    			foundChannel=false;
+	    			nextBrace=false;
+	    		}
+	    	}
+	    	
+	    	/*if (outputStuff && vals++<1000) {
+	    		System.out.print((char)cp);
+	    	}*/
+	    	  if ((char)cp == '{') {
+	    		  braceCount++;
+	    		  //System.out.println("Brace count is "+braceCount+".");
+	    	  } else
+    		  if ((char)cp == '}') {
+	    		  braceCount--;
+	    		  //System.out.println("Brace count is "+braceCount+".");
+	    	  }
+	    	
+		      if (allowed) {
+		    	  sb.append((char) cp);
+		      }
+	     }
+	    sb.append("}");
+	   // System.out.println("=============");
+	    System.out.println(sb.toString());
+	    return sb.toString();
+	  }
 
-	  public static JSONObject readJsonFromUrl(String url) throws IOException, JSONException {
+	  public static JSONObject readJsonFromUrlWithFilter(String url, HashMap<Long,String> filter) throws IOException, JSONException {
-	    InputStream is = new URL(url).openStream();
+	    return readJsonFromUrlWithFilter(url,filter,null,false);
+	  }
+
+	  public static JSONObject readJsonFromFileWithFilter(String file, HashMap<Long,String> filter) throws IOException, JSONException {
+	    InputStream is = new FileInputStream(new File(file));
 	    try {
 	      BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
-	      String jsonText = readAll(rd);
+	      String jsonText = readFilter(rd,filter);
 	      JSONObject json = new JSONObject(jsonText);
 	      jsonText=null;
 	      return json;
@@ -69,6 +168,25 @@ public class FileUtils {
 	    }
 	  }
 
+	  public static JSONObject readJsonFromUrlWithFilter(String url, HashMap<Long,String> filter, String file, boolean writeToFile) throws IOException, JSONException {
+	    InputStream is = new URL(url).openStream();
+	    try {
+	      BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
+	      String jsonText = readFilter(rd,filter);
+	      if (writeToFile) {
+	    	  writetoFile(new String[]{jsonText},file);
+	      }
+	      JSONObject json = new JSONObject(jsonText);
+	      return json;
+	    } finally {
+	      is.close();
+	    }
+	  }
+
+	  public static JSONObject readJsonFromUrl(String url) throws IOException, JSONException {
+	    return readJsonFromUrl(url,null,false);
+	  }
+
 	  public static JSONObject readJsonFromFile(String file) throws IOException, JSONException {
 	    InputStream is = new FileInputStream(new File(file));
 	    try {