C#实现web信息自动抓取
http://tech.ddvip.com 2007年10月05日 社区交流
内容摘要：实现自定义网站信息分类抓取，存入本地数据库、生成静态页面或其它用户定义的信息结构，并下载与信息相关的多媒体文件。
得到要抓取内容的url后，处理该页面：
//获取链接内容并分类处理
public string GetWebContent( string gatherUrl,string subUrl,string subTitle,string b_Content,string e_Content,string b_Filter,string e_Filter,string root )
{
string tmpAllStr = "";
string dfStrB = "";
string dfStrE = "";
string rePicStr = "";//图片返回路径
string reContentStr = "";
string picHtml = "images"; //本地图片路径
string urlch1 ="";
string urlch2 ="";
int pos1 = gatherUrl.IndexOf( "." );
int pos2 = gatherUrl.LastIndexOf( "/" );
if( pos1 < 0 )
{
return "";
}
if( pos2 < 0 )
{
return "";
}
int pos3 = gatherUrl.IndexOf( "/",pos1 );
if ( pos3 < 0 )
{
urlch1 = gatherUrl;
urlch2 = gatherUrl;
}
else
{
urlch1 = gatherUrl.Substring( 0,pos3 );
urlch2 = gatherUrl.Substring( 0,pos2 );
}
tmpAllStr = new PublicFun().Get_Http( subUrl,time1 );
//取稿源
string docFromStr = "";
if ( dfStrB != "" && dfStrE != "" )
{
if ( tmpAllStr != "" )
{
int b_docF = tmpAllStr.IndexOf( dfStrB );
if ( b_docF > 0 )
{
&nbs p; int e_docF = tmpAllStr.IndexOf( dfStrE,b_docF + dfStrB.Length );
&nb sp; if ( e_docF > 0 && e_docF > b_docF && e_docF - b_docF < 20 )
{
&nbs p; docFromStr = tmpAllStr.Substring( b_docF + dfStrB.Length, e_docF - b_docF - dfStrB.Length );
&nb sp; }
}
}
}
//取内容
if ( tmpAllStr != "" )
{
int begin_strnum = tmpAllStr.IndexOf( b_Content );
if ( begin_strnum < 0 )
{
return "";
}
int end_strnum = tmpAllStr.IndexOf( e_Content,begin_strnum + b_Content.Length );
if ( end_strnum < 0 )
{
return "";
}
string sTotalSubM = "";
if ( end_strnum > begin_strnum )
{
sTotalSubM = tmpAllStr.Substring ( begin_strnum,end_strnum - begin_strnum );
}
if ( sTotalSubM == "" )
{
return "";
}
//过滤无用信息
int bfnum = sTotalSubM.IndexOf( b_Filter );
if ( bfnum > -1 )
{
int efnum = sTotalSubM.IndexOf( e_Filter,bfnum );
if ( efnum > -1 )
{
&nbs p; if ( efnum > bfnum )
{
&nbs p; sTotalSubM = sTotalSubM.Substring( 0,bfnum ) + sTotalSubM.Substring( efnum + e_Filter.Length );
&nb sp; }
}
}
//格式化图片标记
while( sTotalSubM.IndexOf( "Src=" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "Src=" ) ) + "src=" + sTotalSubM.Substring( sTotalSubM.IndexOf( "Src=" ) + 4 );
}
while( sTotalSubM.IndexOf( "SRC=" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "SRC=" ) ) + "src=" + sTotalSubM.Substring( sTotalSubM.IndexOf( "SRC=" ) + 4 );
}
while( sTotalSubM.IndexOf( "src='" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "src='" ) ) + "src="" + sTotalSubM.Substring( sTotalSubM.IndexOf( "src='" ) + 5 );
}
//取图片地址
int end_n12 = 0;
int end_nums2 = 0;
int begin_nums2 = sTotalSubM.IndexOf( "src=" );
while( begin_nums2 >= 0 )
{
String tmpStr = sTotalSubM.Substring( begin_nums2 + 4,1 );
if ( tmpStr == """ )
{
&nbs p; end_n12 = begin_nums2 + 5;
}
else
{
&nbs p; end_n12 = begin_nums2 + 4;
}
int end_nums2a = sTotalSubM.IndexOf( " ",end_n12 );
int end_nums2b = sTotalSubM.IndexOf( ">",end_n12 );
if ( end_nums2b < 0 )
{
&nbs p; break;
}
if ( end_nums2a > end_nums2b )
{
&nbs p; end_nums2 = end_nums2b;
}
else
{
&nbs p; if (end_nums2a<0)
{
&nbs p; end_nums2 = end_nums2b;
}
else
& nbsp; {
&nbs p; end_nums2 = end_nums2a;
}
}
tmpStr = sTotalSubM.Substring( end_nums2-1,1 );
if ( tmpStr == """ || tmpStr == "'" )
{
&nbs p; end_nums2 = end_nums2 - 1;
}
string tmpPicStr = sTotalSubM.Substring( end_n12,end_nums2 - end_n12 );
if ( tmpPicStr.IndexOf( "http://" ) < 0 )
{
&nbs p; if ( tmpPicStr.IndexOf( "/" ) == 0 )
{
&nbs p; tmpPicStr = urlch1 + tmpPicStr;
& nbsp; }
else
& nbsp; {
int linshiIntNum = 0;
&nb sp; int flags = 0;
&nb sp; string urlChange = subUrl;
&nbs p; while( tmpPicStr.IndexOf( "../" ) >= 0 )
{
&nbs p; tmpPicStr = tmpPicStr.Substring( tmpPicStr.IndexOf("../") + 3 );
&nb sp; linshiIntNum = linshiIntNum + 1;
&nb sp; flags = flags + 1;
&nb sp; }
while( ( urlChange.LastIndexOf( "/" ) >= 0 ) && ( linshiIntNum >= 0 ) )
{
&nbs p; urlChange = urlChange.Substring( 0,urlChange.LastIndexOf( "/" ) );
&nb sp; linshiIntNum = linshiIntNum - 1;
&nb sp; }
if ( flags == 0 )
{
&nbs p; tmpPicStr = urlch2 + "/" + tmpPicStr;
& nbsp; }
else
& nbsp; {
&nbs p; tmpPicStr = urlChange + "/" + tmpPicStr;
& nbsp; }
}
}
//tmpPicStr = tmpPicStr.ToLower();
string tmpPicStrTmp = tmpPicStr.ToLower ();
//if ( tmpPicStr.IndexOf( ".jpg" ) > 0 || tmpPicStr.IndexOf( ".gif" ) > 0 || tmpPicStr.IndexOf( ".bmp" ) > 0 )
if ( tmpPicStrTmp.IndexOf( ".jpg" ) > 0 || tmpPicStrTmp.IndexOf( ".gif" ) > 0 || tmpPicStrTmp.IndexOf( ".bmp" ) > 0 )
{
&nbs p; rePicStr = rePicStr + "||" + tmpPicStr ;
&nbs p; int flagN2 = tmpPicStr.LastIndexOf( "/" );
&nb sp; string fileN2 = picHtml + tmpPicStr.Substring( flagN2 );
&nb sp; sTotalSubM = sTotalSubM.Substring( 0,end_nums2 ) + ">******" + fileN2 + "******<" + sTotalSubM.Substring( end_nums2 );
&nb sp; begin_nums2 = sTotalSubM.IndexOf( "src=", end_nums2 + fileN2.Length + 22 );
}
else
{
&nbs p; begin_nums2 = sTotalSubM.IndexOf( "src=", end_nums2 + 4 );
}
}
if ( rePicStr.Length > 2 )
rePicStr = rePicStr.Substring(2);
//内容处理 格式化关键标记
while( sTotalSubM.IndexOf( "<P" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<P" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<P" ) + 2 );
}
while( sTotalSubM.IndexOf( "<p" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<p" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<p" ) + 2 );
}
while( sTotalSubM.IndexOf( "</P" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "</P" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "</P" ) + 3 );
}
while( sTotalSubM.IndexOf( "</p" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "</p" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "</p" ) + 3 );
}
while( sTotalSubM.IndexOf( "<br" ) >=0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<br" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<br" ) + 3 );
}
while( sTotalSubM.IndexOf( "<BR" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<BR" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<BR" ) + 3 );
}
while( sTotalSubM.IndexOf( "<Br" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<Br" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<Br" ) + 3 );
}
while( sTotalSubM.IndexOf( "<bR" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<bR" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<bR" ) + 3 );
}
//去除html标记
int linshiInt1 = sTotalSubM.IndexOf( "<" );
int linshiInt2 = sTotalSubM.IndexOf( ">" );
if ( linshiInt2 < linshiInt1 )
{
sTotalSubM = sTotalSubM.Substring( linshiInt2 + 1 );
}
int linshiInt11 = sTotalSubM.LastIndexOf( "<" );
int linshiInt12 = sTotalSubM.LastIndexOf( ">" );
if ( linshiInt12 < linshiInt11 )
{
sTotalSubM = sTotalSubM.Substring( 0,linshiInt12 + 1 );
}
linshiInt1 = sTotalSubM.IndexOf( "<" );
while ( linshiInt1 >= 0 )
{
linshiInt2 = sTotalSubM.IndexOf( ">",linshiInt1 );
if ( linshiInt2 >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,linshiInt1 ) + sTotalSubM.Substring( linshiInt2 + 1 );
}
else
{
&nbs p; sTotalSubM = sTotalSubM.Substring( 0,linshiInt1 );
}
linshiInt1 = sTotalSubM.IndexOf("<");
}
//还原关键标记
int linshiInt3 = 0;
int linshiInt4 = 0;
while( sTotalSubM.IndexOf( "+****+" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "+****+" ) ) + "<br>
" + sTotalSubM.Substring( sTotalSubM.IndexOf( "+****+" ) + 9 );
}
while( sTotalSubM.IndexOf( "|****|" ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "|****|" ) ) + "<br>
" + sTotalSubM.Substring( sTotalSubM.IndexOf( "|****|" ) + 9 );
}
while( sTotalSubM.IndexOf( "******" ) >= 0 )
{
linshiInt3 = sTotalSubM.IndexOf( "******" ) + 9;
linshiInt4 = sTotalSubM.IndexOf( "******",linshiInt3 );
if ( linshiInt4 >= 0 )
{
&nbs p; int tmpPos = sTotalSubM.IndexOf( "******" );
&nb sp; string tmpStr1 = sTotalSubM.Substring( 0,tmpPos );
string tmpStr2 = sTotalSubM.Substring( linshiInt3,linshiInt4 - linshiInt3 );
&nb sp; string tmpStr3 = sTotalSubM.Substring( linshiInt4 + 9 );
&nb sp; sTotalSubM = tmpStr1 + "<img src=" + tmpStr2 + ">" + tmpStr3;
}
else
{
&nbs p; break;
}
}
//去除内容中的标题
if ( sTotalSubM.IndexOf( subTitle ) >= 0 )
{
sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( subTitle ) ) + sTotalSubM.Substring( sTotalSubM.IndexOf( subTitle ) + subTitle.Length );
}
reContentStr = sTotalSubM;
//调用下载图片功能
//下载图片到指定目录
string[] img_Url = new PublicFun().split( rePicStr,"||" );
for ( int i=0;i<img_Url.Length;i++ )
{
if ( img_Url[i] != "" )
{
&nbs p; new PublicFun().Get_Img( img_Url[i],10000,root + "\images\" + img_Url[i].Substring( img_Url[i].LastIndexOf("/")+1 ) );
}
}
}
return reContentStr;
}
责编：豆豆技术应用