欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识!
您现在的位置是: 首页

正则表达式抓取网页信息

程序员文章站 2022-07-13 16:10:52
...

     声明:此正则表达式只适用于.net ,使用的流程为发送http请求返回整个html网页,然后从此html页面抓取想要的数据。

 

 

第一部分:发送httpWebRequest 请求

 

//url 地址
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("URL"));
            HttpWebResponse response = (HttpWebResponse)request.GetResponse();
            //浏览器类型设置
            request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.0.04506; .NET CLR 3.5.21022; .NET CLR 1.0.3705; .NET CLR 1.1.4322)";
            StreamReader reader = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding("UTF-8"));
              //返回的html网页数据
            String htmlStr = reader.ReadToEnd();

  第二部分:根据返回的html获取有用数据,此方法适用于所有想通过ID或Class等等的标签找到html的需求,拿下面一个方法为例

	/// <summary>
        /// 获得颜色
         /// </summary>
        /// <param name="htmlStr"></param>
        /// <returns></returns>
        public String getColor(String htmlStr)
        {
  //获取class为  DetailsC_Sku的html ,还可改为ID的方式   
//string regstr6 = @"<(?<HtmlTag>[\w]+)[^>]*\s[iI][dD]=(?<Quote>";    
string regstr6 = @"<(?<HtmlTag>[\w]+)[^>]*\s[cC][lL][aA][sS][sS]=(?<Quote>";
            string regstr7 = "[\"']?)DetailsC_Sku(?(Quote)";
            string regstr8 = @"\k<Quote>)";
            string regstr9 = "[\"']?[^>]*>";
            string regstr10 = @"((?<Nested><\k<HtmlTag>[^>]*>)|</\k<HtmlTag>>(?<-Nested>)|.*?)*</\k<HtmlTag>>";
            StringBuilder sb2 = new StringBuilder();
            sb2.Append(regstr6);
            sb2.Append(regstr7);
            sb2.Append(regstr8);
            sb2.Append(regstr9);
            sb2.Append(regstr10);
	    //根据正则表达式获取的html
            String sizeHtml = Regex.Match(htmlStr, sb2.ToString(), RegexOptions.Singleline).ToString();
            if (!String.IsNullOrEmpty(sizeHtml))
            {
                String newhtml = htmlStr.Replace(sizeHtml, "");
                string regstr11 = @"<(?<HtmlTag>[\w]+)[^>]*\s[cC][lL][aA][sS][sS]=(?<Quote>";
                string regstr12 = "[\"']?)DetailsC_Sku(?(Quote)";
                string regstr13 = @"\k<Quote>)";
                string regstr14 = "[\"']?[^>]*>";
                string regstr15 = @"((?<Nested><\k<HtmlTag>[^>]*>)|</\k<HtmlTag>>(?<-Nested>)|.*?)*</\k<HtmlTag>>";
                StringBuilder sb3 = new StringBuilder();
                sb3.Append(regstr11);
                sb3.Append(regstr12);
                sb3.Append(regstr13);
                sb3.Append(regstr14);
                sb3.Append(regstr15);
                String colorHtml = Regex.Match(newhtml, sb3.ToString(), RegexOptions.Singleline).ToString();
                if (String.IsNullOrEmpty(colorHtml))
                    return "";

                //找出此colorHtml中的所有a 标签
                Regex regex2 = new Regex(@"<a.*?>[\s\S]*?<\/a>");
                MatchCollection mc2 = regex2.Matches(colorHtml);
                StringBuilder sbs = new StringBuilder();
                //循环找到颜色
                if (mc2.Count > 0)
                {
                    foreach (Match mm in mc2)
                    {
                        sbs.Append(RemoveHtml(mm.Value.ToString())).Append(",");
                    }
                }
                return sbs.ToString();
            }
            return "";
            
        }

 

 

 /// <summary>
       /// 替换字符串中的html标签为空返回标签里的内容
        /// </summary>
        /// <param name="src"></param>
        /// <returns></returns>
        public string RemoveHtml(string src)
        {
            Regex htmlReg = new Regex(@"<[^>]+>", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Regex htmlSpaceReg = new Regex("\\&nbsp\\;", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Regex spaceReg = new Regex("\\s{2,}|\\ \\;", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Regex styleReg = new Regex(@"<style(.*?)</style>", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Regex scriptReg = new Regex(@"<script(.*?)</script>", RegexOptions.Compiled | RegexOptions.IgnoreCase);

            src = styleReg.Replace(src, string.Empty);
            src = scriptReg.Replace(src, string.Empty);
            src = htmlReg.Replace(src, string.Empty);
            src = htmlSpaceReg.Replace(src, " ");
            src = spaceReg.Replace(src, " ");
            return src.Trim();
        }