Java Code Gists

Retrofit Crawler

May 7, 2017December 24, 2016 by T Tak

A simple web crawler that uses a Retrofit service to turn URLs into webpages.

/*
 * Copyright (C) 2016 Square, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.example.retrofit;
 
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import okhttp3.ConnectionPool;
import okhttp3.Dispatcher;
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import okhttp3.ResponseBody;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import retrofit2.Call;
import retrofit2.Callback;
import retrofit2.Converter;
import retrofit2.Response;
import retrofit2.Retrofit;
import retrofit2.http.GET;
import retrofit2.http.Url;
 

public final class Crawler {
  private final Set fetchedUrls = Collections.synchronizedSet(
      new LinkedHashSet());
  private final ConcurrentHashMap hostnames = new ConcurrentHashMap&lt;&gt;();
  private final PageService pageService;
 
  public Crawler(PageService pageService) {
    this.pageService = pageService;
  }
 
  public void crawlPage(HttpUrl url) {
    // Skip hosts that we've visited many times.
    AtomicInteger hostnameCount = new AtomicInteger();
    AtomicInteger previous = hostnames.putIfAbsent(url.host(), hostnameCount);
    if (previous != null) hostnameCount = previous;
    if (hostnameCount.incrementAndGet() &gt; 100) return;
 
    // Asynchronously visit URL.
    pageService.get(url).enqueue(new Callback() {
      @Override public void onResponse(Call call, Response response) {
        if (!response.isSuccessful()) {
          System.out.println(call.request().url() + ": failed: " + response.code());
          return;
        }
 
        // Print this page's URL and title.
        Page page = response.body();
        HttpUrl base = response.raw().request().url();
        System.out.println(base + ": " + page.title);
 
        // Enqueue its links for visiting.
        for (String link : page.links) {
          HttpUrl linkUrl = base.resolve(link);
          if (linkUrl != null &amp;&amp; !fetchedUrls.add(linkUrl)) {
            crawlPage(linkUrl);
          }
        }
      }
 
      @Override public void onFailure(Call call, Throwable t) {
        System.out.println(call.request().url() + ": failed: " + t);
      }
    });
  }
 
  public static void main(String... args) throws Exception {
    Dispatcher dispatcher = new Dispatcher(Executors.newFixedThreadPool(20));
    dispatcher.setMaxRequests(20);
    dispatcher.setMaxRequestsPerHost(1);
 
    OkHttpClient okHttpClient = new OkHttpClient.Builder()
        .dispatcher(dispatcher)
        .connectionPool(new ConnectionPool(100, 30, TimeUnit.SECONDS))
        .build();
 
    Retrofit retrofit = new Retrofit.Builder()
        .baseUrl(HttpUrl.parse("https://example.com/"))
        .addConverterFactory(PageAdapter.FACTORY)
        .client(okHttpClient)
        .build();
 
    PageService pageService = retrofit.create(PageService.class);
 
    Crawler crawler = new Crawler(pageService);
    crawler.crawlPage(HttpUrl.parse(args[0]));
  }
 
  interface PageService {
    @GET Call get(@Url HttpUrl url);
  }
 
  static class Page {
    public final String title;
    public final List links;
 
    public Page(String title, List links) {
      this.title = title;
      this.links = links;
    }
  }
 
  static final class PageAdapter implements Converter {
    static final Converter.Factory FACTORY = new Converter.Factory() {
      @Override public Converter responseBodyConverter(
          Type type, Annotation[] annotations, Retrofit retrofit) {
        if (type == Page.class) return new PageAdapter();
        return null;
      }
    };
 
    @Override public Page convert(ResponseBody responseBody) throws IOException {
      Document document = Jsoup.parse(responseBody.string());
      List links = new ArrayList&lt;&gt;();
      for (Element element : document.select("a[href]")) {
        links.add(element.attr("href"));
      }
      return new Page(document.title(), Collections.unmodifiableList(links));
    }
  }
}

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

* Licensed under the Apache License, Version 2.0 (the "License");

* you may not use this file except in compliance with the License.

* You may obtain a copy of the License at

* http://www.apache.org/licenses/LICENSE-2.0

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

package com.example.retrofit;

import java.io.IOException;

import java.lang.annotation.Annotation;

import java.lang.reflect.Type;

import java.util.ArrayList;

import java.util.Collections;

import java.util.LinkedHashSet;

import java.util.List;

import java.util.Set;

import java.util.concurrent.ConcurrentHashMap;

import java.util.concurrent.Executors;

import java.util.concurrent.TimeUnit;

import java.util.concurrent.atomic.AtomicInteger;

import okhttp3.ConnectionPool;

import okhttp3.Dispatcher;

import okhttp3.HttpUrl;

import okhttp3.OkHttpClient;

import okhttp3.ResponseBody;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import retrofit2.Call;

import retrofit2.Callback;

import retrofit2.Converter;

import retrofit2.Response;

import retrofit2.Retrofit;

import retrofit2.http.GET;

import retrofit2.http.Url;

public final class Crawler {

private final Set fetchedUrls = Collections.synchronizedSet(

new LinkedHashSet());

private final ConcurrentHashMap hostnames = new ConcurrentHashMap<>();

private final PageService pageService;

public Crawler(PageService pageService) {

this.pageService = pageService;

}

public void crawlPage(HttpUrl url) {

// Skip hosts that we've visited many times.

AtomicInteger hostnameCount = new AtomicInteger();

AtomicInteger previous = hostnames.putIfAbsent(url.host(), hostnameCount);

if (previous != null) hostnameCount = previous;

if (hostnameCount.incrementAndGet() > 100) return;

// Asynchronously visit URL.

pageService.get(url).enqueue(new Callback() {

@Override public void onResponse(Call call, Response response) {

if (!response.isSuccessful()) {

System.out.println(call.request().url() + ": failed: " + response.code());

return;

}

// Print this page's URL and title.

Page page = response.body();

HttpUrl base = response.raw().request().url();

System.out.println(base + ": " + page.title);

// Enqueue its links for visiting.

for (String link : page.links) {

HttpUrl linkUrl = base.resolve(link);

if (linkUrl != null && !fetchedUrls.add(linkUrl)) {

crawlPage(linkUrl);

}

@Override public void onFailure(Call call, Throwable t) {

System.out.println(call.request().url() + ": failed: " + t);

}

});

}

public static void main(String... args) throws Exception {

Dispatcher dispatcher = new Dispatcher(Executors.newFixedThreadPool(20));

dispatcher.setMaxRequests(20);

dispatcher.setMaxRequestsPerHost(1);

OkHttpClient okHttpClient = new OkHttpClient.Builder()

.dispatcher(dispatcher)

.connectionPool(new ConnectionPool(100, 30, TimeUnit.SECONDS))

.build();

Retrofit retrofit = new Retrofit.Builder()

.baseUrl(HttpUrl.parse("https://example.com/"))

.addConverterFactory(PageAdapter.FACTORY)

.client(okHttpClient)

.build();

PageService pageService = retrofit.create(PageService.class);

Crawler crawler = new Crawler(pageService);

crawler.crawlPage(HttpUrl.parse(args[0]));

}

interface PageService {

@GET Call get(@Url HttpUrl url);

}

static class Page {

public final String title;

public final List links;

public Page(String title, List links) {

this.title = title;

this.links = links;

}

static final class PageAdapter implements Converter {

static final Converter.Factory FACTORY = new Converter.Factory() {

@Override public Converter responseBodyConverter(

Type type, Annotation[] annotations, Retrofit retrofit) {

if (type == Page.class) return new PageAdapter();

return null;

}

};

@Override public Page convert(ResponseBody responseBody) throws IOException {

Document document = Jsoup.parse(responseBody.string());

List links = new ArrayList<>();

for (Element element : document.select("a[href]")) {

links.add(element.attr("href"));

}

return new Page(document.title(), Collections.unmodifiableList(links));

}

Convert byte to Hexadecimal String and vice versa

May 7, 2017December 24, 2016 by T Tak

Convert byte to Hexadecimal

 public static String bytesToHex(byte[] bytes) {
    final char[] hexArray = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' };
    char[] hexChars = new char[bytes.length * 2];
    int v;
    for (int j = 0; j &lt; bytes.length; j++) {
        v = bytes[j] &amp; 0xFF;
        hexChars[j * 2] = hexArray[v &gt;&gt;&gt; 4];
        hexChars[j * 2 + 1] = hexArray[v &amp; 0x0F];
    }
    return new String(hexChars);
    }

public static String bytesToHex(byte[] bytes) {

final char[] hexArray = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' };

char[] hexChars = new char[bytes.length * 2];

int v;

for (int j = 0; j < bytes.length; j++) {

v = bytes[j] & 0xFF;

hexChars[j * 2] = hexArray[v >>> 4];

hexChars[j * 2 + 1] = hexArray[v & 0x0F];

}

return new String(hexChars);

}

Convert Hexadecimal to byte

public static byte[] hexToBytes(String s) {
    int len = s.length();
    byte[] data = new byte[len / 2];
    for (int i = 0; i &lt; len; i += 2) {
        data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) &lt;&lt; 4) + Character.digit(s.charAt(i + 1), 16));
    }
    return data;
    }

public static byte[] hexToBytes(String s) {

int len = s.length();

byte[] data = new byte[len / 2];

for (int i = 0; i < len; i += 2) {

data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) + Character.digit(s.charAt(i + 1), 16));

}

return data;

}

How to download and Upload a file through SFTP using java.

May 7, 2017December 23, 2016 by T Tak

Here is a code example that shows how to download and upload a file through SFTP in java Download File

import com.jcraft.jsch.Channel;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpException;
/**
 * 
 * @author javagists.com
 *
 */
public class DownloadFileSFTP {
 
    public static void main(String[] args) throws Exception {

 JSch jsch = new JSch();
        Session session = null;
        try {
           session = jsch.getSession("admin", "127.0.0.1", 22);
            session.setConfig("StrictHostKeyChecking", "no");
            session.setPassword("pass");
            session.connect();
            
            Channel channel = session.openChannel("sftp");
            channel.connect();
            ChannelSftp sftpChannel = (ChannelSftp) channel;
            sftpChannel.get("/tmpremote/testDownload.txt", "/tmplocal/testDownload.txt");  
            sftpChannel.exit();
            session.disconnect();
        } catch (JSchException e) {
            e.printStackTrace();  
        } catch (SftpException e) {
            e.printStackTrace();
        }

   }

}

import com.jcraft.jsch.Channel;

import com.jcraft.jsch.ChannelSftp;

import com.jcraft.jsch.JSch;

import com.jcraft.jsch.JSchException;

import com.jcraft.jsch.Session;

import com.jcraft.jsch.SftpException;

/**

* @author javagists.com

public class DownloadFileSFTP {

public static void main(String[] args) throws Exception {

JSch jsch = new JSch();

Session session = null;

try {

session = jsch.getSession("admin", "127.0.0.1", 22);

session.setConfig("StrictHostKeyChecking", "no");

session.setPassword("pass");

session.connect();

Channel channel = session.openChannel("sftp");

channel.connect();

ChannelSftp sftpChannel = (ChannelSftp) channel;

sftpChannel.get("/tmpremote/testDownload.txt", "/tmplocal/testDownload.txt");

sftpChannel.exit();

session.disconnect();

} catch (JSchException e) {

e.printStackTrace();

} catch (SftpException e) {

e.printStackTrace();

}

Upload File

import com.jcraft.jsch.Channel;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpException;
/**
 * 
 * @author javagists.com
 *
 */
public class UploadFileSFTP {
 
 public static void main(String[] args) throws Exception {

  JSch jsch = new JSch();
        Session session = null;
        try {
            session = jsch.getSession("admin", "127.0.0.1", 22);
            session.setConfig("StrictHostKeyChecking", "no");
            session.setPassword("pass");
            session.connect();
            
            Channel channel = session.openChannel("sftp");
            channel.connect();
            ChannelSftp sftpChannel = (ChannelSftp) channel;
            sftpChannel.put("/tmplocal/testUpload.txt", "/tmpremote/testUpload.txt");  
            sftpChannel.exit();
            session.disconnect();
        } catch (JSchException e) {
            e.printStackTrace();  
        } catch (SftpException e) {
            e.printStackTrace();
        }

 }

}

import com.jcraft.jsch.Channel;

import com.jcraft.jsch.ChannelSftp;

import com.jcraft.jsch.JSch;

import com.jcraft.jsch.JSchException;

import com.jcraft.jsch.Session;

import com.jcraft.jsch.SftpException;

/**

* @author javagists.com

public class UploadFileSFTP {

public static void main(String[] args) throws Exception {

JSch jsch = new JSch();

Session session = null;

try {

session = jsch.getSession("admin", "127.0.0.1", 22);

session.setConfig("StrictHostKeyChecking", "no");

session.setPassword("pass");

session.connect();

Channel channel = session.openChannel("sftp");

channel.connect();

ChannelSftp sftpChannel = (ChannelSftp) channel;

sftpChannel.put("/tmplocal/testUpload.txt", "/tmpremote/testUpload.txt");

sftpChannel.exit();

session.disconnect();

} catch (JSchException e) {

e.printStackTrace();

} catch (SftpException e) {

e.printStackTrace();

}