溫馨提示×
您好，登錄后才能下訂單哦！
點擊 登錄注冊 即表示同意《億速云用戶服務條款》
您好，登錄后才能下訂單哦！
這篇文章將為大家詳細講解有關如何使用java實現登錄之后抓取數據功能，小編覺得挺實用的，因此分享給大家做個參考，希望大家閱讀完這篇文章后可以有所收獲。
具體內容如下：
首先需要一個jsoup的jar包，我用的1.6.0。。下載地址為：http://pan.baidu.com/s/1mgqOuHa
1，獲取網頁內容（核心代碼，技術有限沒封裝）。
2，登錄之后抓取網頁數據（如何在請求中攜帶cookie）。
3，獲取網站的ajax請求方法（返回json）。
以上這三點我就用一個類全部包含（比較糙望見諒，直接copy代碼過去，應該就可以用）
一，這個類分別有這上面的1,2,3三中方法，直接main方法可以進行測試
package com.minxinloan.black.web.utils;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.StringTokenizer;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class CookieUtil {
public final static String CONTENT_TYPE = "Content-Type";
public static void main(String[] args) {
//String loginURL = "http://www.p2peye.com/member.php?mod=logging&action=login&loginsubmit=yes&loginhash=Lsc66&username=puqiuxiaomao&password=a1234567";
String listURL = "http://www.p2peye.com/blacklist.php?p=2";
String logURL = "http://www.p2peye.com/member.php";
//********************************需要登錄的*************************************************
try {
Connection.Response res =
Jsoup.connect(logURL)
.data("mod","logging"
,"action","login"
,"loginsubmit","yes"
,"loginhash","Lsc66"
,"username","puqiuxiaomao"
,"password","a1234567")
.method(Method.POST)
.execute();
//這兒的SESSIONID需要根據要登錄的目標網站設置的session Cookie名字而定
Connection con=Jsoup.connect(listURL);
//設置訪問形式（電腦訪問，手機訪問）：直接百度都參數設置
con.header("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)");
//把登錄信息的cookies保存如map對象里面
Map <String,String> map=res.cookies();
Iterator<Entry<String,String>> it =map.entrySet().iterator();
while(it.hasNext()){
Entry<String,String> en= it.next();
//把登錄的信息放入請求里面
con =con.cookie(en.getKey(), en.getValue());
}
//再次獲取Document對象。
Document objectDoc = con.get();
Elements elements = objectDoc.getAllElements();//獲取這個連接返回頁面的源碼內容（不是源碼跟源碼差不多）
for (Element element : elements) {
//element是迭代出來的標簽：如：<div><span></span></div>
Elements elements2= element.getAllElements();//
for (Element element2 : elements2) {
element2.text();
element2.attr("href");//獲取標簽屬性。element2代表a標簽：href代表屬性
element2.text();//獲取標簽文本
}
}
//********************************不需要登錄的*************************************************
String URL = "http://www.p2peye.com/blacklist.php?p=2";
Document conTemp = Jsoup.connect(URL).get();
Elements elementsTemps = conTemp.getAllElements();
for (Element elementsTemp : elementsTemps) {
elementsTemp.text();
elementsTemp.attr("href");//獲取標簽屬性。element2代表a標簽：href代表屬性
elementsTemp.text();//獲取標簽文本
}
//********************************ajax方法獲取內容。。。*************************************************。
HttpURLConnection connection = null;
BufferedReader reader = null;
try {
StringBuffer sb = new StringBuffer();
URL getUrl = new URL(URL);
connection = (HttpURLConnection)getUrl.openConnection();
reader = new BufferedReader(new InputStreamReader(
connection.getInputStream(),"utf-8"));
String lines;
while ((lines = reader.readLine()) != null) {
sb.append(lines);
};
List<Map<String, Object>> list = parseJSON2List(sb.toString());//json轉換成list
} catch (Exception e) {
} finally{
if(reader!=null)
try {
reader.close();
} catch (IOException e) {
}
// 斷開連接
connection.disconnect();
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static Map<String, Object> parseJSON2Map(String jsonStr){
Map<String, Object> map = new HashMap<String, Object>();
//最外層解析
JSONObject json = JSONObject.fromObject(jsonStr);
for(Object k : json.keySet()){
Object v = json.get(k);
//如果內層還是數組的話，繼續解析
if(v instanceof JSONArray){
List<Map<String, Object>> list = new ArrayList<Map<String,Object>>();
Iterator<JSONObject> it = ((JSONArray)v).iterator();
while(it.hasNext()){
JSONObject json2 = it.next();
list.add(parseJSON2Map(json2.toString()));
}
map.put(k.toString(), list);
} else {
map.put(k.toString(), v);
}
}
return map;
}
public static List<Map<String, Object>> parseJSON2List(String jsonStr){
JSONArray jsonArr = JSONArray.fromObject(jsonStr);
List<Map<String, Object>> list = new ArrayList<Map<String,Object>>();
Iterator<JSONObject> it = jsonArr.iterator();
while(it.hasNext()){
JSONObject json2 = it.next();
list.add(parseJSON2Map(json2.toString()));
}
return list;
}
}二，這個是獲取驗證碼的類，可以研究下。（但你要要分析出網站的驗證碼的請求地址）
package com.minxinloan.black.web.utils;
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
public class Utils {//解析驗證碼的
public static Content getRandom(String method, String sUrl,// 要解析的url
Map<String, String> paramMap, // 存放用戶名和密碼的map
Map<String, String> requestHeaderMap,// 存放COOKIE的map
boolean isOnlyReturnHeader, String path) {
Content content = null;
HttpURLConnection httpUrlConnection = null;
InputStream in = null;
try {
URL url = new URL(sUrl);
boolean isPost = "POST".equals(method);
if (method == null
|| (!"GET".equalsIgnoreCase(method) && !"POST"
.equalsIgnoreCase(method))) {
method = "POST";
}
URL resolvedURL = url;
URLConnection urlConnection = resolvedURL.openConnection();
httpUrlConnection = (HttpURLConnection) urlConnection;
httpUrlConnection.setRequestMethod(method);
httpUrlConnection.setRequestProperty("Accept-Language",
"zh-cn,zh;q=0.5");
// Do not follow redirects, We will handle redirects ourself
httpUrlConnection.setInstanceFollowRedirects(false);
httpUrlConnection.setDoOutput(true);
httpUrlConnection.setDoInput(true);
httpUrlConnection.setConnectTimeout(5000);
httpUrlConnection.setReadTimeout(5000);
httpUrlConnection.setUseCaches(false);
httpUrlConnection.setDefaultUseCaches(false);
httpUrlConnection.connect();
int responseCode = httpUrlConnection.getResponseCode();
if (responseCode == HttpURLConnection.HTTP_OK
|| responseCode == HttpURLConnection.HTTP_CREATED) {
byte[] bytes = new byte[0];
if (!isOnlyReturnHeader) {
DataInputStream ins = new DataInputStream(
httpUrlConnection.getInputStream());
// 驗證碼的位置
DataOutputStream out = new DataOutputStream(
new FileOutputStream(path + "/code.bmp"));
byte[] buffer = new byte[4096];
int count = 0;
while ((count = ins.read(buffer)) > 0) {
out.write(buffer, 0, count);
}
out.close();
ins.close();
}
String encoding = null;
if (encoding == null) {
encoding = getEncodingFromContentType(httpUrlConnection
.getHeaderField(""));
}
content = new Content(sUrl, new String(bytes, encoding),
httpUrlConnection.getHeaderFields());
}
} catch (Exception e) {
return null;
} finally {
if (httpUrlConnection != null) {
httpUrlConnection.disconnect();
}
}
return content;
}
public static String getEncodingFromContentType(String contentType) {
String encoding = null;
if (contentType == null) {
return null;
}
StringTokenizer tok = new StringTokenizer(contentType, ";");
if (tok.hasMoreTokens()) {
tok.nextToken();
while (tok.hasMoreTokens()) {
String assignment = tok.nextToken().trim();
int eqIdx = assignment.indexOf('=');
if (eqIdx != -1) {
String varName = assignment.substring(0, eqIdx).trim();
if ("charset".equalsIgnoreCase(varName)) {
String varValue = assignment.substring(eqIdx + 1)
.trim();
if (varValue.startsWith("\"")
&& varValue.endsWith("\"")) {
// substring works on indices
varValue = varValue.substring(1,
varValue.length() - 1);
}
if (Charset.isSupported(varValue)) {
encoding = varValue;
}
}
}
}
}
if (encoding == null) {
return "UTF-8";
}
return encoding;
}
// 這個是輸出
public static boolean inFile(String content, String path) {
PrintWriter out = null;
File file = new File(path);
try {
if (!file.exists()) {
file.createNewFile();
}
out = new PrintWriter(new FileWriter(file));
out.write(content);
out.flush();
return true;
} catch (Exception e) {
e.printStackTrace();
} finally {
out.close();
}
return false;
}
public static String getHtmlReadLine(String httpurl) {
String CurrentLine = "";
String TotalString = "";
InputStream urlStream;
String content = "";
try {
URL url = new URL(httpurl);
HttpURLConnection connection = (HttpURLConnection) url
.openConnection();
connection.connect();
System.out.println(connection.getResponseCode());
urlStream = connection.getInputStream();
BufferedReader reader = new BufferedReader(
new InputStreamReader(urlStream, "utf-8"));
while ((CurrentLine = reader.readLine()) != null) {
TotalString += CurrentLine + "\n";
}
content = TotalString;
} catch (Exception e) {
}
return content;
}
}
class Content {
private String url;
private String body;
private Map<String, List<String>> m_mHeaders = new HashMap<String, List<String>>();
public Content(String url, String body, Map<String, List<String>> headers) {
this.url = url;
this.body = body;
this.m_mHeaders = headers;
}
public String getUrl() {
return url;
}
public String getBody() {
return body;
}
public Map<String, List<String>> getHeaders() {
return m_mHeaders;
}
}關于“如何使用java實現登錄之后抓取數據功能”這篇文章就分享到這里了，希望以上內容可以對大家有一定的幫助，使各位可以學到更多知識，如果覺得文章不錯，請把它分享出去讓更多的人看到。
免責聲明：本站發布的內容（圖片、視頻和文字）以原創、轉載和分享為主，文章觀點不代表本網站立場，如果涉及侵權請聯系站長郵箱：is@yisu.com進行舉報，并提供相關證據，一經查實，將立刻刪除涉嫌侵權內容。
