溫馨提示×
您好，登錄后才能下訂單哦！
點擊 登錄注冊 即表示同意《億速云用戶服務條款》
您好，登錄后才能下訂單哦！
這篇文章將為大家詳細講解有關如何驗證數據庫中URL的有效性，文章內容質量較高，因此小編分享給大家做個參考，希望大家閱讀完這篇文章后對相關知識有一定的了解。
曲庫中一些歌曲的URL雖然存在,但是根據URL已經下載不到音樂了.
Nginx顯示404錯誤.
驗證數據庫中歌曲的URL是否能夠下載
首先,先把數據庫中的歌曲URL導出到文件.如下格式(歌曲ID,音樂地址類型,路徑)
1000;AccompanimentURL ;/00/00/00001000_accompaniment.m4a
然后使用程序掃描URL
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.7</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.7</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.0.Final</version>
</dependency>
程序比較渣..以后得學習一下對象封裝
<ol start="1" class="dp-j" font-style:normal;font-variant-ligatures:normal;font-variant-caps:normal;font-weight:normal;letter-spacing:normal;orphans:2;text-align:left;text-indent:0px;text-transform:none;white-space:normal;widows:2;word-spacing:0px;-webkit-text-stroke-width:0px;text-decoration-style:initial;text-decoration-color:initial;">
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.LineBasedFrameDecoder;
import io.netty.handler.codec.string.StringDecoder;
public class URLChecker {
public static void main(String[] args) {
String sourceFile="F:\\normal.txt";
String resultFile="F:\\result.csv";
new URLInput(sourceFile, resultFile);
}
}
class URLConnection extends Thread{
@Override
public void run() {
while(true){
int start=count.get();
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
int end=count.get();
System.out.println("每秒檢查:"+(end-start));
}
}
Semaphore sem = new Semaphore(20);
String[] hosts = new String[5];
AtomicInteger count = new AtomicInteger();
EventLoopGroup group = new NioEventLoopGroup(3);
URLInput input;
URLOutput output;
URLConnection(URLInput input,URLOutput writer) {
hosts[0] = "172.16.1.151";
hosts[1] = "172.16.1.152";
hosts[2] = "172.16.1.153";
hosts[3] = "172.16.1.154";
hosts[4] = "172.16.1.155";
this.output=writer;
this.input=input;
}
public void connection(final Map<String, String> map)
throws InterruptedException {
sem.acquire();
int index = count.getAndIncrement();
Bootstrap boot = new Bootstrap();
boot.group(group).channel(NioSocketChannel.class).handler(new ChannelInitializer() {
@Override
protected void initChannel(Channel ch) throws Exception {
ch.pipeline().addLast(new LineBasedFrameDecoder(409600));
ch.pipeline().addLast(new StringDecoder());
ch.pipeline().addLast(new HttpClientHandler(map, sem,input,output));
}
});
boot.connect(hosts[index % hosts.length], 80);
}
}
class HttpClientHandler extends ChannelInboundHandlerAdapter {
StringBuffer sb = new StringBuffer(512);
Map<String, String> map = new HashMap<String, String>();
Semaphore sem;
URLInput input;
URLOutput writer;
public HttpClientHandler(Map<String, String> map, Semaphore sem, URLInput input, URLOutput writer) {
this.map = map;
this.sem = sem;
this.writer=writer;
this.input=input;
}
@Override
public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
sem.release();
ctx.close();
writer.addSuccURL(map);
}
@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
StringBuilder sb = new StringBuilder();
sb.append("HEAD " + map.get("url") + " HTTP/1.0\r\n");
sb.append("HOST:" + 80 + "\r\n");
sb.append("Accept:*/*\r\n");
sb.append("\r\n");
ByteBuf bb = Unpooled.copiedBuffer(sb.toString().getBytes("utf8"));
ctx.writeAndFlush(bb);
}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
String content = (String) msg;
if (content.contains(":")) {
String[] s = content.split(":");
map.put(s[0].trim(), s[1].trim());
} else if (content.startsWith("HTTP/1.1")) {
map.put("httpcode", content.replaceAll("HTTP/1.1 ", ""));
}
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
cause.printStackTrace();
ctx.close();
input.addFailUrl(map);
sem.release();
}
}
class URLInput {
URLConnection urlcon;
URLInput(String sourceFile, String resultFile) {
URLOutput output=new URLOutput(resultFile);
urlcon=new URLConnection(this, output);
output.start();
urlcon.start();
try {
init(resultFile);
read(sourceFile);
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
/**
* 初始化已經處理的文件,用于中斷處理后的恢復運行
*
* @param resultFile
* @throws IOException
*/
private void init(String resultFile) throws IOException {
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(resultFile)));
String row = null;
while ((row = br.readLine()) != null) {
String[] data = row.split(",");
set.add(data[0]);
}
br.close();
}
public void addFailUrl(Map<String, String> map) {
failq.add(map);
}
final BlockingQueue<Map<String, String>> failq = new LinkedBlockingQueue<Map<String, String>>();
Set<String> set = new HashSet<String>();
private void read(String sourceFile) throws IOException, InterruptedException {
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFile)));
String row = null;
while ((row = br.readLine()) != null) {
while (failq.size() != 0) {
final Map<String, String> m = failq.take();
urlcon.connection(m);
}
String[] data = row.split(";");
final Map<String, String> map = new HashMap<String, String>();
map.put("songid", data[0]);
map.put("type", data[1]);
map.put("url", data[2]);
if (!set.contains(data[0])) {
urlcon.connection(map);
}
}
br.close();
System.out.println("Finish!!");
}
}
class URLOutput extends Thread {
BlockingQueue<Map<String, String>> succq = new LinkedBlockingQueue<Map<String, String>>();
String resultFile;
public void addSuccURL(Map<String, String> map) {
succq.add(map);
}
public URLOutput(String resultFile) {
this.resultFile = resultFile;
}
@Override
public void run() {
Map<String, String> map = null;
FileWriter fw = null;
try {
fw = new FileWriter(resultFile, true);
while ((map = succq.take()) != null) {
fw.write(map.get("songid") + "," + map.get("type") + "," + map.get("url") + "," + map.get("httpcode")
+ "," + map.get("Content-Length") + "\n");
}
} catch (InterruptedException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
fw.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
關于如何驗證數據庫中URL的有效性就分享到這里了，希望以上內容可以對大家有一定的幫助，可以學到更多知識。如果覺得文章不錯，可以把它分享出去讓更多的人看到。
免責聲明：本站發布的內容（圖片、視頻和文字）以原創、轉載和分享為主，文章觀點不代表本網站立場，如果涉及侵權請聯系站長郵箱：is@yisu.com進行舉報，并提供相關證據，一經查實，將立刻刪除涉嫌侵權內容。
