C#落到实处web音讯自动抓取

NetBug–爬虫 V1.02付出笔记

背景

趁着Internet的推广,互联网音信正以超级高的进程提升,在如此多多少中找到本身索要的音讯是一件很麻烦的事体,找到须求的新闻后怎样收获也是件劳心的事。这就须要Internet消息抓取程序来代表人工的操作。

所谓Internet新闻抓取程序,正是程序会根据客户的重大词或首要网址来搜罗相应的新闻,并提必要客商想要的音讯格式。

音信量的充实会带给消息网址发表职员事业量的大幅度增加,为落到实处消息宣布种类落实音讯自

动公布、减弱专门的学业人士工作量、即时追踪最新消息,就须要活动消息提供程序,因而Internet消息抓取程序现身。

目标

心想事成自定义网站新闻分类抓取,存入本地数据库、生成静态页面或其余客商定义的消息布局,并下载与音信有关的多媒体文件。

开发

l 指标站点布局分析

本步骤是标准抓打消息个举足轻重。

第一要选取立异频率高的页面做为抓取地址,然后深入分析要抓取内容页面url特点。

下一场解析要抓取音信页面的要素性子,举个例子标题位置,内容位置等,获得稳固标志点。

将以上新闻写成团结的配置文件或存到数据库中。

各样网址都须要解析,写出单身的布局文件,供抓取程序行使。

l 新闻提取

基于计划文件获得要抓取页面url,使用HttpWebRequest类获取内容:

//获取http页面函数

public string Get_Http(string a_strUrl,int timeout)

{

string strResult ;

try

{

HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(a_strUrl)
;

myReq.Timeout = timeout;

HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse();

Stream myStream = HttpWResp.GetResponseStream () ;

StreamReader sr = new StreamReader(myStream , Encoding.Default);

StringBuilder strBuilder = new StringBuilder();

while (-1 != sr.Peek())

{

strBuilder.Append(sr.ReadLine()+”rn”);

}

strResult = strBuilder.ToString();

}

catch(Exception exp)

{

strResult = “错误:” + exp.Message ;

}

return strResult ;

}

获得页面内容后,分析页面中接二连三地址取到要抓取的url:

//管理页面标题和链接

public string SniffWebUrl( string urlStr,string blockB,string blockE )

{

string urlch1 = “”;

string urlch2 = “”;

int end_n1 = 0;

int end_nums = 0;

int end_nums1 = 0;

int end_nums2 = 0;

int end_nums3 = 0;

string reUTStr = “”;

string reTitle = “”;

string ret = “”;

try

{

int pos01 = urlStr.IndexOf( “.” );

int pos02 = urlStr.LastIndexOf( “/” );

if( pos01 < 0 )

{

return “”;

}

if( pos02 < 0 )

{

return “”;

}

int pos03 = urlStr.IndexOf( “/”,pos01 );

if ( pos03 < 0 )

{

urlch1 = urlStr;

urlch2 = urlStr;

}

else

{

urlch1 = urlStr.Substring( 0,pos03 );

urlch2 = urlStr.Substring( 0,pos02 );

}

string tmpAllStr = new PublicFun().Get_Http( urlStr ,time1);

int pos1 = tmpAllStr.IndexOf( blockB );

int pos2 = tmpAllStr.IndexOf( blockE,pos1 + blockB.Length );

if ( pos1>0 && pos2>0 && pos2>pos1 )

{

ret = tmpAllStr.Substring( pos1 + blockB.Length,pos2 – pos1 –
blockB.Length );

ret = ret.Substring( ret.IndexOf( “
while( ret.IndexOf( “= 0 )

{

ret = ret.Substring( 0,ret.IndexOf( “
}

while( ret.IndexOf( “=0 )

{

ret = ret.Substring( 0,ret.IndexOf( “
}

while( ret.IndexOf( “Href=” ) >=0 )

{

ret = ret.Substring( 0,ret.IndexOf( “Href=”%20))%20+%20″href=” +
ret.Substring( ret.IndexOf( “Href=” ) + 5 );

}

while( ret.IndexOf( “HREF=” ) >=0 )

{

ret = ret.Substring( 0,ret.IndexOf( “HREF=”%20))%20+%20″href=” +
ret.Substring( ret.IndexOf( “HREF=” ) + 5 );

}

while( ret.IndexOf( “href='” ) >=0 )

{

ret = ret.Substring( 0,ret.IndexOf( “href='”%20))%20+%20″href=”” +
ret.Substring( ret.IndexOf( “href='” ) + 6 );

}

}

tmpAllStr = ret;

int begin_nums = tmpAllStr.IndexOf( “href=” );

while ( begin_nums >= 0 )

{

string tmpStrA = “”;

string tmpStrB = tmpAllStr.Substring( begin_nums + 5,1 );

if ( tmpStrB == “”” )

{

end_n1 = begin_nums + 6;

if ( ( end_n1 + 1 ) > tmpAllStr.Length )

{

return “”;

}

tmpStrA = tmpAllStr.Substring( begin_nums+6,1 );

}

else

{

end_n1 = begin_nums + 5;

tmpStrA = tmpStrB;

}

if ( tmpStrA == “#” )

{

tmpAllStr = tmpAllStr.Substring( end_n1 );

begin_nums = tmpAllStr.IndexOf( “href=” );

}

else

{

end_nums1 = tmpAllStr.IndexOf( ” “,end_n1 );

end_nums2 = tmpAllStr.IndexOf( “>”,end_n1 );

end_nums3 = tmpAllStr.IndexOf( ”

if ( ( end_nums3 >= 0 ) && ( end_nums2 >= 0 ) )

{

reTitle = tmpAllStr.Substring( end_nums2 + 1,end_nums3 – end_nums2 –
1 );

if ( end_nums1 > end_nums2 )

{

end_nums = end_nums2;

}

else

{

if ( end_nums1 < 0 )

{

end_nums = end_nums2;

}

else

{

end_nums = end_nums1;

}

}

string str4 = tmpAllStr.Substring( end_nums-1, end_nums – end_nums +
1 );

if ( str4 ==””” || str4 == “‘” )

{

end_nums = end_nums – 1;

}

string sTotalOne = tmpAllStr.Substring( end_n1,end_nums – end_n1 );

if ( sTotalOne.IndexOf( “http://” ) <0 )

{

if ( sTotalOne.IndexOf( “/” ) == 0 )

{

sTotalOne = urlch1 + sTotalOne;

}

else

{

int linshiIntNum = 0;

int flags = 0;

string urlChange = urlStr;;

while( sTotalOne.IndexOf( “../” ) >= 0 )

{

sTotalOne = sTotalOne.Substring( sTotalOne.IndexOf( “../” ) + 3 );

linshiIntNum = linshiIntNum + 1;

flags = flags +1;

}

while( ( urlChange.LastIndexOf( “/” ) >= 0 ) && ( linshiIntNum >=
0 ) )

{

urlChange = urlChange.Substring( 0,urlChange.LastIndexOf( “/” ) );

linshiIntNum = linshiIntNum – 1;

}

if ( flags == 0 )

{

sTotalOne = urlch2 + “/” + sTotalOne;

}

else

{

sTotalOne = urlChange + “/” + sTotalOne;

}

}

}

reUTStr = reUTStr + new PublicFun().RemoveHtmlCode( reTitle ) +
sTotalOne;

tmpAllStr = tmpAllStr.Substring( end_nums3 + 4 );

begin_nums = tmpAllStr.IndexOf( “href=” );

}

else

{

begin_nums = -1;

}

}

}

return reUTStr;

}

catch( Exception e)

{

return “”;

}

}

取得要抓取内容的url后,管理该页面:

//获取链接内容并分类管理

public string GetWebContent( string gatherUrl,string subUrl,string
subTitle,string b_Content,string e_Content,string b_Filter,string
e_Filter,string root )

{

string tmpAllStr = “”;

string dfStrB = “”;

string dfStrE = “”;

string rePicStr = “”;//图片再次来到路径

string reContentStr = “”;

string picHtml = “images”; //当地图片路径

string urlch1 =””;

string urlch2 =””;

int pos1 = gatherUrl.IndexOf( “.” );

int pos2 = gatherUrl.LastIndexOf( “/” );

if( pos1 < 0 )

{

return “”;

}

if( pos2 < 0 )

{

return “”;

}

int pos3 = gatherUrl.IndexOf( “/”,pos1 );

if ( pos3 < 0 )

{

urlch1 = gatherUrl;

urlch2 = gatherUrl;

}

else

{

urlch1 = gatherUrl.Substring( 0,pos3 );

urlch2 = gatherUrl.Substring( 0,pos2 );

}

tmpAllStr = new PublicFun().Get_Http( subUrl,time1 );

//取稿源

string docFromStr = “”;

if ( dfStrB != “” && dfStrE != “” )

{

if ( tmpAllStr != “” )

{

int b_docF = tmpAllStr.IndexOf( dfStrB );

if ( b_docF > 0 )

{

int e_docF = tmpAllStr.IndexOf( dfStrE,b_docF + dfStrB.Length );

if ( e_docF > 0 && e_docF > b_docF && e_docF – b_docF < 20
)

{

docFromStr = tmpAllStr.Substring( b_docF + dfStrB.Length, e_docF –
b_docF – dfStrB.Length );

}

}

}

}

//取内容

if ( tmpAllStr != “” )

{

int begin_strnum = tmpAllStr.IndexOf( b_Content );

if ( begin_strnum < 0 )

{

return “”;

}

int end_strnum = tmpAllStr.IndexOf( e_Content,begin_strnum +
b_Content.Length );

if ( end_strnum < 0 )

{

return “”;

}

string sTotalSubM = “”;

if ( end_strnum > begin_strnum )

{

sTotalSubM = tmpAllStr.Substring ( begin_strnum,end_strnum –
begin_strnum );

}

if ( sTotalSubM == “” )

{

return “”;

}

//过滤无用音信

int bfnum = sTotalSubM.IndexOf( b_Filter );

if ( bfnum > -1 )

{

int efnum = sTotalSubM.IndexOf( e_Filter,bfnum );

if ( efnum > -1 )

{

if ( efnum > bfnum )

{

sTotalSubM = sTotalSubM.Substring( 0,bfnum ) + sTotalSubM.Substring(
efnum + e_Filter.Length );

}

}

}

//格式化图片标志

while( sTotalSubM.IndexOf( “Src=” ) >= 0 )

{

sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( “Src=” ) ) +
“src=”%20+%20sTotalSubM.Substring(%20sTotalSubM.IndexOf(%20″Src=” ) + 4 );

}

while( sTotalSubM.IndexOf( “SRC=” ) >= 0 )

{

sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( “SRC=” ) ) +
“src=”%20+%20sTotalSubM.Substring(%20sTotalSubM.IndexOf(%20″SRC=” ) + 4 );

}

while( sTotalSubM.IndexOf( “src='” ) >= 0 )

{

sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( “src='” ) ) +
“src=””%20+%20sTotalSubM.Substring(%20sTotalSubM.IndexOf(%20″src='” ) + 5 );

}

//取图片地址

int end_n12 = 0;

int end_nums2 = 0;

int begin_nums2 = sTotalSubM.IndexOf( “src=” );

while( begin_nums2 >= 0 )

{

String tmpStr = sTotalSubM.Substring( begin_nums2 + 4,1 );

if ( tmpStr == “”” )

{

end_n12 = begin_nums2 + 5;

}

else

{

end_n12 = begin_nums2 + 4;

}

int end_nums2a = sTotalSubM.IndexOf( ” “,end_n12 );

int end_nums2b = sTotalSubM.IndexOf( “>”,end_n12 );

if ( end_nums2b < 0 )

{

break;

}

if ( end_nums2a > end_nums2b )

{

end_nums2 = end_nums2b;

}

else

{

if (end_nums2a<0)

{

end_nums2 = end_nums2b;

}

else

{

end_nums2 = end_nums2a;

}

}

tmpStr = sTotalSubM.Substring( end_nums2-1,1 );

if ( tmpStr == “”” || tmpStr == “‘” )

{

end_nums2 = end_nums2 – 1;

}

string tmpPicStr = sTotalSubM.Substring( end_n12,end_nums2 – end_n12
);

if ( tmpPicStr.IndexOf( “http://” ) < 0 )

{

if ( tmpPicStr.IndexOf( “/” ) == 0 )

{

tmpPicStr = urlch1 + tmpPicStr;

}

else

{

int linshiIntNum = 0;

int flags = 0;

string urlChange = subUrl;

while( tmpPicStr.IndexOf( “../” ) >= 0 )

{

tmpPicStr = tmpPicStr.Substring( tmpPicStr.IndexOf(“../”) + 3 );

linshiIntNum = linshiIntNum + 1;

flags = flags + 1;

}

while( ( urlChange.LastIndexOf( “/” ) >= 0 ) && ( linshiIntNum >=
0 ) )

{

urlChange = urlChange.Substring( 0,urlChange.LastIndexOf( “/” ) );

linshiIntNum = linshiIntNum – 1;

}

if ( flags == 0 )

{

tmpPicStr = urlch2 + “/” + tmpPicStr;

}

else

{

tmpPicStr = urlChange + “/” + tmpPicStr;

}

}

}

//tmpPicStr = tmpPicStr.ToLower();

string tmpPicStrTmp = tmpPicStr.ToLower();

//if ( tmpPicStr.IndexOf( “.jpg” ) > 0 || tmpPicStr.IndexOf( “.gif” )
> 0 || tmpPicStr.IndexOf( “.bmp” ) > 0 )

if ( tmpPicStrTmp.IndexOf( “.jpg” ) > 0 || tmpPicStrTmp.IndexOf(
“.gif” ) > 0 || tmpPicStrTmp.IndexOf( “.bmp” ) > 0 )

{

rePicStr = rePicStr + “||” + tmpPicStr ;

int flagN2 = tmpPicStr.LastIndexOf( “/” );

string fileN2 = picHtml + tmpPicStr.Substring( flagN2 );

sTotalSubM = sTotalSubM.Substring( 0,end_nums2 ) + “>******” +
fileN2 + “******

begin_nums2 = sTotalSubM.IndexOf( “src=”, end_nums2 + fileN2.Length +
22 );

}

else

{

begin_nums2 = sTotalSubM.IndexOf( “src=”, end_nums2 + 4 );

}

}

if ( rePicStr.Length > 2 )

rePicStr = rePicStr.Substring(2);

//内容管理 格式化关键标志

while( sTotalSubM.IndexOf( “

= 0 )

{

sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( “

}

while( sTotalSubM.IndexOf( “

= 0 )

{

sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( “

}

while( sTotalSubM.IndexOf( “

= 0 )

{

sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( “
}

while( sTotalSubM.IndexOf( “= 0 )

{

sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( “
}

while( sTotalSubM.IndexOf( “
=0 )

{

sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( ”

}

while( sTotalSubM.IndexOf( “
= 0 )

{

sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( ”

}

while( sTotalSubM.IndexOf( “
= 0 )

{

sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( ”

}

while( sTotalSubM.IndexOf( “
= 0 )

{

sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( ”

}

//去除html标记

int linshiInt1 = sTotalSubM.IndexOf( “
int linshiInt2 = sTotalSubM.IndexOf( “>” );

if ( linshiInt2 < linshiInt1 )

{

sTotalSubM = sTotalSubM.Substring( linshiInt2 + 1 );

}

int linshiInt11 = sTotalSubM.LastIndexOf( “
int linshiInt12 = sTotalSubM.LastIndexOf( “>” );

if ( linshiInt12 < linshiInt11 )

{

sTotalSubM = sTotalSubM.Substring( 0,linshiInt12 + 1 );

}

linshiInt1 = sTotalSubM.IndexOf( “
while ( linshiInt1 >= 0 )

{

linshiInt2 = sTotalSubM.IndexOf( “>”,linshiInt1 );

if ( linshiInt2 >= 0 )

{

sTotalSubM = sTotalSubM.Substring( 0,linshiInt1 ) +
sTotalSubM.Substring( linshiInt2 + 1 );

}

else

{

sTotalSubM = sTotalSubM.Substring( 0,linshiInt1 );

}

linshiInt1 = sTotalSubM.IndexOf(“
}

//还原第一标记

int linshiInt3 = 0;

int linshiInt4 = 0;

while( sTotalSubM.IndexOf( “+****+” ) >= 0 )

{

sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( “+****+” )
) + “
n”%20+%20sTotalSubM.Substring(%20sTotalSubM.IndexOf(%20″+****+” ) + 9 );

}

while( sTotalSubM.IndexOf( “|****|” ) >= 0 )

{

sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( “|****|” )
) + “
n”%20+%20sTotalSubM.Substring(%20sTotalSubM.IndexOf(%20″|****|” ) + 9 );

}

while( sTotalSubM.IndexOf( “******” ) >= 0 )

{

linshiInt3 = sTotalSubM.IndexOf( “******” ) + 9;

linshiInt4 = sTotalSubM.IndexOf( “******”,linshiInt3 );

if ( linshiInt4 >= 0 )

{

int tmpPos = sTotalSubM.IndexOf( “******” );

string tmpStr1 = sTotalSubM.Substring( 0,tmpPos );

string tmpStr2 = sTotalSubM.Substring( linshiInt3,linshiInt4 –
linshiInt3 );

string tmpStr3 = sTotalSubM.Substring( linshiInt4 + 9 );

sTotalSubM = tmpStr1 +
图片 1” + tmpStr3;

}

else

{

break;

}

}

//去除内容中的题目

if ( sTotalSubM.IndexOf( subTitle ) >= 0 )

{

sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( subTitle ) ) +
sTotalSubM.Substring( sTotalSubM.IndexOf( subTitle ) + subTitle.Length
);

}

reContentStr = sTotalSubM;

//调用下载图片效用

//下载图片到钦点目录

string[] img_Url = new PublicFun().split( rePicStr,”||” );

for ( int i=0;i
{

if ( img_Url[i] != “” )

{

new PublicFun().Get_Img( img_Url[i],10000,root + “\images\” +
img_Url[i].Substring( img_Url[i].LastIndexOf(“/”)+1 ) );

}

}

}

return reContentStr;

}

以上办法重返要收获的音讯,富含标题内容,图片地址等。

下载页面中图纸:

//下载图片

public void Get_Img(string a_strUrl,int timeout,string filepath)

{

try

{

HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(a_strUrl)
;

myReq.Timeout = timeout;

HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse();

Stream myStream = HttpWResp.GetResponseStream () ;

Bitmap map = new Bitmap( myStream );

PictureBox picB = new PictureBox();

picB.Image = (Image)map;

string path = filepath.Substring( 0,filepath.LastIndexOf( “\” ) );

if (!Directory.Exists(path))

{

CreateDir( path );

}

picB.Image.Save(filepath);

}

catch(Exception exp)

{

string ss = exp.Message;

WriteLog( filepath.Substring(0,filepath.LastIndexOf(“\”)) +
“\error.log”,a_strUrl + “–” + ss + “rn”);

}

}

l 保存文件或入库

地方获得的消息方可按本身的须要保存。

****设计的时候从不采纳url按档案的次序循环抓取,那样定义抓取url功效越来越高,速度更加快。

测量检验程序下载:

如有建议请发送EMail:xiancai@qianlong.com 或msn yutao728@hotmail.com

注:此版本只提供静态文件存款和储蓄作用,不提供数据库接口,不提供自定义网站功效。

本程序运转须要先安装.net 框架1.1。

相关文章