Concurrent Web Requests with Thread Pooling

If we want to fetch the content of some web URLs concurrently we can use multi thread theqniques but something important is that when we are going to make a lot of web requests we should not consume all the CPU resource!
So one of the good designs could be implementing some kind of thread pools.

Download Source

Test Class

package com.ehsunbehravesh.asyncwebreader;

import java.util.HashMap;
import java.util.Observable;
import java.util.Observer;

public class Test2 {

  public static void main(String[] args) throws Exception {
    AsyncWebReader webReader = new AsyncWebReader(5/* threads */, new String[]{
              "http://www.google.com",
              "http://www.yahoo.com",
              "http://www.live.com",
              "http://www.wikipedia.com",
              "http://www.facebook.com",
              "http://www.khorasannews.com",
              "http://www.fcbarcelona.com",
              "http://www.khorasannews.com",
            });

    webReader.addObserver(new Observer() {
      @Override
      public void update(Observable o, Object arg) {
        if (arg instanceof Exception) {
          Exception ex = (Exception) arg;
          System.out.println(ex.getMessage());
        } /*else if (arg instanceof List) {
          List vals = (List) arg;
          System.out.println(vals.get(0) + ": " + vals.get(1));
        } */else if (arg instanceof Object[]) {
          Object[] objects = (Object[]) arg;
          HashMap result = (HashMap) objects[0];
          String[] success = (String[]) objects[1];
          String[] fail = (String[]) objects[2];

          System.out.println("Failds");
          for (int i = 0; i < fail.length; i++) {
            String string = fail[i];
            System.out.println(string);
          }

          System.out.println("-----------");
          System.out.println("success");
          for (int i = 0; i < success.length; i++) {
            String string = success[i];
            System.out.println(string);
          }
          
          System.out.println("\n\nresult of Google: ");
          System.out.println(result.remove("http://www.google.com"));
        }
      }
    });
    Thread t = new Thread(webReader);
    t.start();
    t.join();
  }
}


AsyncWebReader Class

package com.ehsunbehravesh.asyncwebreader;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Observable;
import java.util.Observer;

public class AsyncWebReader extends Observable implements Runnable, Observer {
  
  private int coundOfThreads;
  private WebReader[] readers;
  private int indicator;
  private String[] urlAddresses;
  private ArrayList successfulAddresses;
  private ArrayList failedAddresses;
  private HashMap result;
  
  private boolean running;
  
  public AsyncWebReader(int coundOfThreads, String[] urlAddresses) throws Exception {
    this.coundOfThreads = coundOfThreads;
    this.urlAddresses = urlAddresses;
    if (coundOfThreads <= 0) {
      throw new Exception("Count of threads should be at least 1!");
    }
    indicator = 0;
  }
  
  
  @Override
  public void run() {
    successfulAddresses = new ArrayList<>();
    failedAddresses = new ArrayList<>();
    result = new HashMap<>();
    running = true;
    readers = new WebReader[Math.min(coundOfThreads, urlAddresses.length)];
    
    for (int i = 0; i < readers.length; i++) {
      readers[i] = new WebReader(urlAddresses[indicator++]);
      readers[i].addObserver(this);
      new Thread(readers[i]).start();
    }
    
    /* wait until all urls get fetched */
    
    while (running && 
            successfulAddresses.size() + failedAddresses.size() 
            < urlAddresses.length) {
      try {        
        Thread.sleep(500);        
      } catch (InterruptedException ex) {
        setChanged();
        notifyObservers(ex);
      }
    }
        
    
    String[] successfulAddressesArray = new String[successfulAddresses.size()];
    successfulAddressesArray = successfulAddresses.toArray(successfulAddressesArray);
    
    String[] failedAddressesArray = new String[failedAddresses.size()];
    failedAddressesArray = failedAddresses.toArray(failedAddressesArray);
    
    setChanged();
    notifyObservers(new Object[] {result, successfulAddressesArray, failedAddressesArray});    
  }
  
  @Override
  public synchronized void update(Observable o, Object arg) {
    WebReader reader = (WebReader) o;
    if (arg instanceof Exception) {      
      failedAddresses.add(reader.getUrlAddress());
    } else if (arg instanceof String) {      
      String content = (String) arg;
      String urlAddress = reader.getUrlAddress();
      result.put(urlAddress, content);
      successfulAddresses.add(urlAddress);
      setChanged();
      List list = new ArrayList<>();
      list.add(urlAddress);
      list.add(content);
      notifyObservers(list);
    }
    
    if (indicator < urlAddresses.length) {
      for (int i = 0; i < readers.length; i++) {
        WebReader currentReader = readers[i];
        if (currentReader == reader) {
          readers[i] = new WebReader(urlAddresses[indicator++]);
          readers[i].addObserver(this);
          new Thread(readers[i]).start();
          break;
        }
      }
    }
  }
  
  public void stop() {
    running = false;
  }  
}


WebReader Class

package com.ehsunbehravesh.asyncwebreader;

import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.Observable;

/**
 * http://codetoearn.blogspot.com
 *
 * @author ehsun7b
 */
public class WebReader extends Observable implements Runnable {

  private String urlAddress;  
  private boolean finished;

  public WebReader(String urlAddress) {
    this.urlAddress = urlAddress;
  }

  @Override
  public void run() {
    finished = false;
    InputStreamReader isr = null;
    try {
      URL url = new URL(urlAddress);
      URLConnection yc = url.openConnection();
      isr = new InputStreamReader(yc.getInputStream());

      char[] buffer = new char[1024];
      int read = 0;
      StringBuilder doc = new StringBuilder();
      while ((read = isr.read(buffer)) > 0) {
        doc.append(buffer, 0, read);
      }
      String content = doc.toString();
      setChanged();
      notifyObservers(content);
    } catch (Exception e) {
      setChanged();
      notifyObservers(e);
    } finally {
      if (isr != null) {
        try {
          isr.close();
        } catch (IOException ex) {
          setChanged();
          notifyObservers(ex);
        }
      }
      finished = true;
    }
  }

  public String getUrlAddress() {
    return urlAddress;
  }    

  public boolean isFinished() {
    return finished;
  }    
}

Swing LinkLabel


It would be very helpful to have a label that can work link HTML A tag. The following child of JLabel have this functionality and the hover color can be customized also.


Download Source


package com.blogspot.codetoearn.linklabel;

import java.awt.Color;
import java.awt.Cursor;
import java.awt.event.MouseEvent;
import java.awt.event.MouseListener;
import java.io.IOException;
import java.net.URISyntaxException;
import javax.swing.Icon;
import javax.swing.JLabel;

/**
 * http://codetoearn.blogspot.com/
 *
 * @author ehsun7b
 */
public class LinkLabel extends JLabel {

  private String url;
  private Color hoverColor = Color.blue;
  private Color forgroundColor;

  public LinkLabel(String url, String text, Icon icon, int horizontalAlignment) {
    super(text, icon, horizontalAlignment);
    this.url = url;
    init();
  }

  public LinkLabel(String url, String text, int horizontalAlignment) {
    super(text, horizontalAlignment);
    this.url = url;
    init();
  }

  public LinkLabel(String url, String text) {
    super(text);
    this.url = url;
    init();
  }

  public LinkLabel(String url, Icon image, int horizontalAlignment) {
    super(image, horizontalAlignment);
    this.url = url;
    init();
  }

  public LinkLabel(String url, Icon image) {
    super(image);
    this.url = url;
    init();
  }

  public LinkLabel(String url) {
    this.url = url;
    init();
  }

  private void init() {
    addMouseListener(new MouseListener() {
      @Override
      public void mouseClicked(MouseEvent e) {
        openUrl();
      }

      @Override
      public void mousePressed(MouseEvent e) {
      }

      @Override
      public void mouseReleased(MouseEvent e) {
      }

      @Override
      public void mouseEntered(MouseEvent e) {
        if (forgroundColor == null) {
          forgroundColor = getForeground();
        }
        setForeground(hoverColor);
      }

      @Override
      public void mouseExited(MouseEvent e) {
        setForeground(forgroundColor);
      }
    });
    
    setCursor(new Cursor(Cursor.HAND_CURSOR));
  }

  private void openUrl() {
    openURL(url);
  }

  public static void openURL(String url) {
    if (!java.awt.Desktop.isDesktopSupported()) {
      return;
    }

    java.awt.Desktop desktop = java.awt.Desktop.getDesktop();

    if (!desktop.isSupported(java.awt.Desktop.Action.BROWSE)) {
      return;
    }

    try {
      java.net.URI uri = new java.net.URI(url);
      desktop.browse(uri);
    } catch (URISyntaxException | IOException e) {
      System.err.println(e.getMessage());
    }
  }

  public String getUrl() {
    return url;
  }

  public void setUrl(String url) {
    this.url = url;
  }

  public Color getHoverColor() {
    return hoverColor;
  }

  public void setHoverColor(Color hoverColor) {
    this.hoverColor = hoverColor;
  }

  public Color getForgroundColor() {
    return forgroundColor;
  }

  public void setForgroundColor(Color forgroundColor) {
    this.forgroundColor = forgroundColor;
  }
}

Swing Fantasy Checkbox with Customized Icons


This customized Checkbox let you have different labels for select or unselected states and you can have different icons also. If you want to have the same label at both states, you can pass the same strings for both to the constructor.


Download Source


package com.blogspot.codetoearn.fantasycheckbox;

import java.awt.FlowLayout;
import java.awt.event.MouseEvent;
import java.awt.event.MouseListener;
import javax.swing.ImageIcon;
import javax.swing.JComponent;
import javax.swing.JLabel;

/**
 * http://codetoearn.blogspot.com/
 *
 * @author ehsun7b
 */
public class FantasyCheckbox extends JComponent implements MouseListener {

  private boolean selected;
  private String selectedLabel;
  private String unSelectedLabel;
  private ImageIcon selectedIcon;
  private ImageIcon unSelectedIcon;
  private JLabel lblIcon;
  private JLabel lblText;

  public FantasyCheckbox(String selectedLabel, String unSelectedLabel, 
          ImageIcon selectedIcon, ImageIcon unSelectedIcon) {
    this.selectedLabel = selectedLabel;
    this.unSelectedLabel = unSelectedLabel;
    this.selectedIcon = selectedIcon;
    this.unSelectedIcon = unSelectedIcon;
    
    createComponents();
  }  
  
  private void createComponents() {
    selected = false;    
    lblIcon = new JLabel(unSelectedIcon);
    lblText = new JLabel(unSelectedLabel);
    setLayout(new FlowLayout());
    add(lblIcon);
    add(lblText);
    lblIcon.addMouseListener(this);
    lblText.addMouseListener(this);
  }  

  private void updateLabels() {
    lblText.setText(selected ? selectedLabel : unSelectedLabel);
    lblIcon.setIcon(selected ? selectedIcon : unSelectedIcon);
  }

  @Override
  public void mouseClicked(MouseEvent e) {
    Object source = e.getSource();
    if (source == lblIcon || source == lblText) {
      selected = !selected;
      updateLabels();      
    }
  }

  @Override
  public void mousePressed(MouseEvent e) {
    
  }

  @Override
  public void mouseReleased(MouseEvent e) {
    
  }

  @Override
  public void mouseEntered(MouseEvent e) {
    
  }

  @Override
  public void mouseExited(MouseEvent e) {
    
  }

  @Override
  public void addMouseListener(MouseListener listener) {
    super.addMouseListener(listener);
    lblIcon.addMouseListener(listener);
    lblText.addMouseListener(listener);
  }
  
  @Override
  public void removeMouseListener(MouseListener listener) {
    super.removeMouseListener(listener);
    lblIcon.removeMouseListener(listener);
    lblText.removeMouseListener(listener);
  }
  
  public boolean isSelected() {
    return selected;
  }

  public void setSelected(boolean selected) {
    this.selected = selected;
    updateLabels();
  }

  public String getSelectedLabel() {
    return selectedLabel;
  }

  public String getUnSelectedLabel() {
    return unSelectedLabel;
  }

  public ImageIcon getSelectedIcon() {
    return selectedIcon;
  }

  public ImageIcon getUnSelectedIcon() {
    return unSelectedIcon;
  }

  public JLabel getLblIcon() {
    return lblIcon;
  }

  public JLabel getLblText() {
    return lblText;
  }

  public void setSelectedLabel(String selectedLabel) {
    this.selectedLabel = selectedLabel;
    updateLabels();
  }

  public void setUnSelectedLabel(String unSelectedLabel) {
    this.unSelectedLabel = unSelectedLabel;
    updateLabels();
  }

  public void setSelectedIcon(ImageIcon selectedIcon) {
    this.selectedIcon = selectedIcon;
    updateLabels();
  }

  public void setUnSelectedIcon(ImageIcon unSelectedIcon) {
    this.unSelectedIcon = unSelectedIcon;
    updateLabels();
  }

  public void setLblIcon(JLabel lblIcon) {
    this.lblIcon = lblIcon;
    updateLabels();
  }

  public void setLblText(JLabel lblText) {
    this.lblText = lblText;
    updateLabels();    
  }
    
}

IP 2 Location in Java

This class shows you how to capture some information including IP, Zip code, ISP and ... using screen scraping technique.

This class is based on the service: www.ip2location.com

IP2Location class
package com.blogspot.codetoearn;

import java.io.IOException;
import java.util.HashMap;
import java.util.Observable;
import java.util.Observer;
import java.util.Set;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * This class is based on http://www.ip2location.com/ and uses a screen scraping
 * technique to capture the information. And always uses the current IP address
 * of the machine. The syntax is based on JDK 7. and uses JSOUP library:
 * http://jsoup.org/ Please note that any change in the layout of
 * www.ip2location.com home page will make troubles for this class.
 *
 * @author ehsun7b
 */
public class IP2Location extends Observable implements Runnable {

  private HashMap information;

  public IP2Location() {
    information = new HashMap<>();
  }

  @Override
  public void run() {
    try {
      Document doc = Jsoup.connect("http://www.ip2location.com/").get();
      Elements elements = doc.select("table.table");
      if (!elements.isEmpty()) {
        Element table = elements.get(0);
        elements = table.select("tbody");
        if (!elements.isEmpty()) {
          Element tbody = elements.get(0);
          Elements lbls = tbody.select("label");

          for (int i = 0; i < lbls.size(); i += 2) {
            Element lblKey = lbls.get(i);
            Element lblValue = lbls.get(i + 1);
            
            String key = lblKey.text();
            String value = lblValue.text();
            
            information.put(key, value);
          }
        }
      }

      setChanged();
      notifyObservers(information);
    } catch (IOException ex) {
      setChanged();
      notifyObservers(ex);
    }
  }

  public static void main(String[] args) {
    IP2Location ip2Location = new IP2Location();
    ip2Location.addObserver(new Observer() {

      @Override
      public void update(Observable o, Object arg) {
        if (arg instanceof HashMap) {
          HashMap result = (HashMap) arg;
          Set keySet = result.keySet();
          for (String key : keySet) {
            String value = result.get(key);
            System.out.println(key + ": " + value);
          }
        }
      }
    });
    
    Thread t = new Thread(ip2Location);    
    t.start();    
  }
}

Note that any change in the layout of the IP2Location website may cause some bug in this code.

Download the IP2Location class here!

Multi Thread TCP Socket Programming

In the previous tutorial we have learned how to implement a very basic TCP communication between a client and a server.
But as you mentioned the server is capable to handle only one client at the time.

So, we show you how to write a more advanced multi-threaded Server to handle several clients simultaneously.

Doing some tasks simultaneously (handling several clients) means that we should design our server in a multi-threaded fashion.

Handling a clients means:
  • sending message to the client
  • receiving clients messages
And we should do this for more than one client at the time plus we still should listen to the port for potential new clients!

Listening to the port will happen in the main Server program thread but for handling individual clients we write the CliendHandler class which implements runnable interface.

In java for creating a thread we have 2 ways:
  1. Extending Thread class (our class inherits from Thread class)
  2. Implementing Runnable interface

We choose the second way in this example since it is a cleaner approach (my opinion).

ClientHandler class
package com.blogspot.codetoearn.advancedtcpsocket.server;

import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;

/**
 * http://codetoearn.blogspot.com/
 *
 * @author ehsun7b
 */
public class ClientHandler implements Runnable {

  private Socket clientSocket;

  public ClientHandler(Socket clientSocket) {
    this.clientSocket = clientSocket;
  }    

  @Override
  public void run() {
    try {
      /* getting input and output streams of the client socket */
      OutputStream outputStream = clientSocket.getOutputStream();
      InputStream inputStream = clientSocket.getInputStream();

      /* sending welcome message to the client */
      String message = "Welcome! You are connected.\n";
      outputStream.write(message.getBytes());
      outputStream.flush();

      /* getting the client reply */
      int character = inputStream.read();

      while (character != -1) {
        System.out.print((char) character);
        character = inputStream.read();
      }

      inputStream.close();
      outputStream.close();
    } catch (Exception ex) {
      System.out.println("Error: " + ex.getMessage());
    }
  }
}

So in the server class, the ServerSocket object accepts any incoming connection and create on ClientHandler object and give it the new socket.

Server class
package com.blogspot.codetoearn.advancedtcpsocket.server;

import java.io.IOException;
import java.net.ServerSocket;
import java.net.Socket;

/**
 * http://codetoearn.blogspot.com/ 
 * @author ehsun7b
 */
public class Server {

  public static final int port = 12345;
  private ServerSocket serverSocket;

  public void listen() {
    try {
      /* creating the serverSocket object */
      serverSocket = new ServerSocket(port);
      
      while (true) {
        Socket socket = serverSocket.accept();
        /* creating a client handler and give the socket to it*/
        ClientHandler clientHandler = new ClientHandler(socket);
        Thread thread = new Thread(clientHandler);
        thread.start();
      }
    } catch (IOException ex) {
      System.out.println("Error: " + ex.getMessage());
    }
  }
  
  public static void main(String[] args) {
    new Server().listen();
  }
}

And finally our client which a simple TCP socket client like the one we have seen in the last example. But we use some simple mechanism to generate 100 random messages to the server from each Client object, to have a simple simulation.

Client class:

package com.blogspot.codetoearn.advancedtcpsocket.client;

import com.blogspot.codetoearn.advancedtcpsocket.utils.Utils;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;

/**
 * http://codetoearn.blogspot.com/
 * @author ehsun7b
 */
public class Client {
  
  private int port;
  private String host;
  private String name;
  private Socket socket;

  public Client(int port, String host, String name) {
    this.port = port;
    this.host = host;
    this.name = name;
  }
  
  public void Connect() {
    try {
      /* creating the socket and trying to connect */
      socket = new Socket(host, port);

      /* getting input and output streams of the socket */
      OutputStream outputStream = socket.getOutputStream();
      InputStream inputStream = socket.getInputStream();

      /* getting the server message */
      int character = inputStream.read();

      while (character != -1 && character != '\n') {
        System.out.print((char) character);
        character = inputStream.read();        
      }
      
      /* sending 100 random messages to the server */
      String[] messages = Utils.randomMessages(100);
      
      for (int i = 0; i < messages.length; i++) {
        String msg = name + ": " + messages[i];        
        outputStream.write(msg.getBytes());
        outputStream.flush();
        Thread.sleep(1000); // wait for 1 second
      }

      inputStream.close();
      outputStream.close();
    } catch (Exception ex) {
      System.out.println("Error: " + ex.getMessage());
    }
  }
  
  /* client program */ 
  public static void main(String[] args) {
    new Thread(new Runnable() {

      @Override
      public void run() {
        new Client(12345, "localhost", "client1").Connect();
      }
    }).start();
    
    new Thread(new Runnable() {

      @Override
      public void run() {
        new Client(12345, "localhost", "client2").Connect();
      }
    }).start();
  }
}

Download the complete code here!

TCP Socket Example - Basics

TCP Socket is a full duplex communication way between client and server. By full duplex it means that both client and server can send and receive at the same time.
The data is transferred as bytes but object which is serializable can be transferred because serializations means that the object convert to a piece of string and strings are arrays of characters which themselves consist of bytes.

So, we can transfer almost everything using TCP sockets.

In this example, we follow this scenario:
  1. We create a server socket which is going to listen to the port number 12345
  2. The server can respond to only one request.
  3. When a client try to connect to the server on port 12345, the server accept the connection and send the client a welcome message.
  4. The client app will show the server welcome message on the terminal and send a message back in reply.
  5. The server will show the client message and close the connection.
So in our server class, we instantiate a ServerSocket object and ask it to listen to the port 12345 and accept the first request. Then we get the client socket input and output streams and send/receive messages to/from client.  

Server Class
package tcp.socket.basic;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.ServerSocket;
import java.net.Socket;

/**
 * http://codetoearn.blogspot.com/
 *
 * @author ehsun7b
 */
public class Server {

  public static final int port = 12345;
  private ServerSocket serverSocket;

  public void listen() {
    try {
      /* creating the serverSocket object */
      serverSocket = new ServerSocket(port);

      /* serverSocket waits for one client */
      Socket clientSocket = serverSocket.accept();

      /* getting input and output streams of the client socket */
      OutputStream outputStream = clientSocket.getOutputStream();
      InputStream inputStream = clientSocket.getInputStream();

      /* sending welcome message to the client */
      String message = "Welcome! You are connected to the "
              + serverSocket.getInetAddress()
              + " on port " + port + "\n";
      outputStream.write(message.getBytes());
      outputStream.flush();

      /* getting the client reply */
      int character = inputStream.read();

      while (character != -1 && character != '\n') {
        System.out.print((char) character);
        character = inputStream.read();        
      }

      inputStream.close();
      outputStream.close();
    } catch (IOException ex) {
      System.out.println("Error: " + ex.getMessage());
    }
  }

  /* creating new server and call it's listen method */
  public static void main(String[] args) {
    new Server().listen();
  }
}

And on the client side we are doing the reverse order. We first get the server welcome message and show it and then we send a message to the server as a reply.

Client Class 
package tcp.socket.basic;

import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;

/**
 * http://codetoearn.blogspot.com/
 *
 * @author ehsun7b
 */
public class Client {

  public static final int port = 12345;
  public static final String host = "127.0.0.1";
  private Socket socket;

  public void connect() {
    try {
      /* creating the socket and trying to connect */
      socket = new Socket(host, port);

      /* getting input and output streams of the socket */
      OutputStream outputStream = socket.getOutputStream();
      InputStream inputStream = socket.getInputStream();


      /* getting the server message */
      int character = inputStream.read();

      while (character != -1 && character != '\n') {
        System.out.print((char) character);
        character = inputStream.read();        
      }

      /* replying the server */
      String message = "I am the client and thanks for accepting me.\n";
      outputStream.write(message.getBytes());
      outputStream.flush();

      inputStream.close();
      outputStream.close();
    } catch (Exception ex) {
      System.out.println("Error: " + ex.getMessage());
    }
  }
  
  public static void main(String[] args) {
    new Client().connect();
  }
}

Note that the server program should be executed first.

Download the programs here.