• .NET 爬虫从入门到入狱

    目录

    前言

    1.💡使用HttpClient爬取数据

    2.🚀模拟User-Agent

    3.🤵使用HTML解析库

    3.👌前端Price显示

    4.🌱运行实例 获取金价Au

    5.🧾使用正则表达式解析

    6.💫获取BTC价格

     7.✨获取CSDN热点

    8.🎃 获取编程语言排行榜

    9.🖥️获取小破站热门

    10.⭐获取某音热门 

    前言

    爬虫可以用于外汇，期货，基金，货币，比分，电商，文章等信息的采集。通过数据分析，竞品分析，为商业决策提供数据支持。

    金价暴涨，抓取下今日的金价。

    1.💡使用HttpClient爬取数据

    1. try
    2. {
    3.     // 创建HttpClient实例
    4.     using (var httpClient = new HttpClient())
    5.     {
    6.         //模拟User-Agent
    7.         httpClient.DefaultRequestHeaders.Add("User-Agent", GetUserAgent());
    8.         // 发送GET请求并获取响应 xxx.com是某网站的页面~（保护）
    9.         var response = await httpClient.GetAsync("https://xxx.com");
    10.         response.EnsureSuccessStatusCode();
    11.         var htmlContent = await response.Content.ReadAsStringAsync();
    12.  
    13.       
    14.     }
    15. }
    16. catch (HttpRequestException ex)
    17. {
    18.     // 处理请求异常
    19.     ViewBag.Error = "Failed to retrieve price data: " + ex.Message;
    20. }

    2.🚀模拟User-Agent

    1.   // 生成随机 User-Agent
    2.   private string GetUserAgent()
    3.   {
    4.       string[] userAgents = new string[]
    5.       {
    6.           "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36",
    7.           "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36",
    8.           // 
    9.       };
    10.  
    11.       Random random = new Random();
    12.       int index = random.Next(userAgents.Length);
    13.       return userAgents[index];
    14.   }

    3.🤵使用HTML解析库

    （HtmlAgilityPack）提取价格

    1.   // 使用HtmlAgilityPack解析HTML
    2.         var htmlDocument = new HtmlDocument();
    3.         htmlDocument.LoadHtml(htmlContent);
    4.  
    5.         // 使用XPath表达式选择指定的元素
    6.         var trElement = htmlDocument.DocumentNode.SelectSingleNode("//tr[contains(@class, 'border_ea') and contains(@class, 'noTop_border')]");
    7.  
    8.         if (trElement != null)
    9.         {
    10.             // 获取包含价格的元素
    11.             var priceCell = trElement.SelectSingleNode(".//td[@align='center'][2]");
    12.  
    13.             if (priceCell != null)
    14.             {
    15.                 // 提取价格
    16.                 string price = priceCell.InnerText.Trim();
    17.  
    18.                 // 将价格传递给视图
    19.                 ViewBag.Price = price;
    20.             }
    21.             else
    22.             {
    23.                 ViewBag.Error = "Price cell not found in table row.";
    24.             }
    25.         }
    26.         else
    27.         {
    28.             ViewBag.Error = "Table row not found.";
    29.         }

    也可以用正则表达式提取

    3.👌前端Price显示

    1.     <h1>Au99.99 Price: @ViewBag.Priceh1>
    2.     <h2>@ViewBag.Errorh2>

    4.🌱运行实例 获取金价Au

    也可以通过修改规则取实时Pt的价格

    1.       // 使用XPath表达式选择指定的元素
    2.       var trElements = htmlDocument.DocumentNode.SelectNodes("//tr[contains(@class, 'border_ea') and contains(@class, 'noTop_border ')]");
    3.  
    4.       if (trElements != null)
    5.       {
    6.           foreach (var trElement in trElements)
    7.           {
    8.               // 获取包含产品名称和价格的元素
    9.               var tdElements = trElement.SelectNodes("./td");
    10.               if (tdElements != null && tdElements.Count >= 5)
    11.               {
    12.                   // 获取产品名称、最新价、最高价、最低价和今开盘价格
    13.                   string productName = tdElements[0].InnerText.Trim();
    14.                   string latestPrice = tdElements[1].InnerText.Trim();
    15.                   string highestPrice = tdElements[2].InnerText.Trim();
    16.                   string lowestPrice = tdElements[3].InnerText.Trim();
    17.                   string openingPrice = tdElements[4].InnerText.Trim();
    18.  
    19.                   // 检查产品名称是否为Au99.99或Pt99.95
    20.                   if (productName == "Au99.99")
    21.                   {
    22.                       // 将Au99.99价格信息传递给视图
    23.                       ViewBag.AuLatestPrice = latestPrice;
    24.                       ViewBag.AuHighestPrice = highestPrice;
    25.                       ViewBag.AuLowestPrice = lowestPrice;
    26.                       ViewBag.AuOpeningPrice = openingPrice;
    27.                   }
    28.                   else if (productName == "Pt99.95")
    29.                   {
    30.                       // 将Pt99.95价格信息传递给视图
    31.                       ViewBag.PtLatestPrice = latestPrice;
    32.                       ViewBag.PtHighestPrice = highestPrice;
    33.                       ViewBag.PtLowestPrice = lowestPrice;
    34.                       ViewBag.PtOpeningPrice = openingPrice;
    35.                   }
    36.               }
    37.           }
    38.       }

    结果如下：

    5.🧾使用正则表达式解析

    通过httpClient请求

    1.      public async Task<ActionResult> Index()
    2.      {
    3.          // 获取当前时间的时间戳
    4.          long timestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds();
    5.  
    6.          // 构建URL
    7.          string url = $"http://www.xxx.cn/xx.js?t={timestamp}";
    8.  
    9.          // 创建HttpClient实例
    10.          using (var httpClient = new HttpClient())
    11.          {
    12.              // 设置 User-Agent
    13.              httpClient.DefaultRequestHeaders.Add("User-Agent", GetUserAgent());
    14.  
    15.              try
    16.              {
    17.                  // 发送GET请求
    18.                  var response = await httpClient.GetAsync(url);
    19.                  response.EnsureSuccessStatusCode();
    20.  
    21.                  // 读取返回的数据
    22.                  var responseData = await response.Content.ReadAsStringAsync();
    23.  
    24.                  // 解析返回的数据
    25.                  var price = ParseGoldPrice(responseData);
    26.  
    27.                  // 将价格传递给视图
    28.                  ViewBag.Price = price;
    29.              }
    30.              catch (HttpRequestException ex)
    31.              {
    32.                  // 处理请求异常
    33.                  ViewBag.Error = "Failed to retrieve gold price data.";
    34.              }
    35.          }
    36.  
    37.          // 返回视图
    38.          return View();
    39.      }

    处理返回价格 

    通过正则表达式匹配数据

    1.   private decimal ParseGoldPrice(string responseData)
    2.   {
    3.  
    4.  
    5.       string price = "";
    6.       decimal price2 = 0;
    7.   
    8.       string data = responseData;// 
    9.  
    10.       // 匹配价格的正则表达式
    11.       string pattern = @"var hq_str_gds_AUTD=""([^""]+)"";";
    12.  
    13.       // 使用正则表达式匹配数据
    14.       Match match = Regex.Match(data, pattern);
    15.  
    16.       if (match.Success)
    17.       {
    18.           // 获取匹配到的价格数据
    19.           string priceData = match.Groups[1].Value;
    20.  
    21.           // 使用逗号分割数据，取第一个元素作为价格
    22.           string[] priceParts = priceData.Split(',');
    23.           price = priceParts[0];
    24.  
    25.           // 将字符串价格转换为decimal类型
    26.           price2 = decimal.Parse(price);
    27.  
    28.           // 输出提取到的价格
    29.           Console.WriteLine("Gold Price: " + price2);
    30.       }
    31.       else
    32.       {
    33.           Console.WriteLine("Price not found in data.");
    34.       }
    35.  
    36.       // decimal price = decimal.Parse(priceString);
    37.      
    38.       return price2;
    39.   }

    6.💫获取BTC价格

    Headers模拟cookie获取BTC价格

    1.  
    2.  httpClient.DefaultRequestHeaders.Add("User-Agent", GetUserAgent());
    3.  
    4.  httpClient.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7");
    5.  httpClient.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9");
    6.  httpClient.DefaultRequestHeaders.Add("Cache-Control", "max-age=0");
    7. //cookie
    8.  httpClient.DefaultRequestHeaders.Add("Cookie", "__51uvsct__3ExGyQaAoNSqsSUY=1; __51vcke__3ExGyQaAoNSqsSUY=c91184d5-8826-5ea8-8ddc-a0f3b85c9470; __51vuft__3ExGyQaAoNSqsSUY=1713248151570; PHPSESSID=a6ff05p79bbf3ot4ohphein0e1; Hm_lvt_1605442054faab140873b7c14e40c707=1713248152; Hm_lvt_4820535acfded9186d46b7ae0c829918=1713248152; __vtins__3ExGyQaAoNSqsSUY=%7B%22sid%22%3A%20%22757fcc49-fe4e-5985-bc10-5a0f6ef6d6ba%22%2C%20%22vd%22%3A%206%2C%20%22stt%22%3A%201175754%2C%20%22dr%22%3A%20271898%2C%20%22expires%22%3A%201713251127320%2C%20%22ct%22%3A%201713249327320%7D; Hm_lpvt_1605442054faab140873b7c14e40c707=1713249327; Hm_lpvt_4820535acfded9186d46b7ae0c829918=1713249328");
    9.  httpClient.DefaultRequestHeaders.Add("Sec-Ch-Ua", "\"Google Chrome\";v=\"123\", \"Not:A-Brand\";v=\"8\", \"Chromium\";v=\"123\"");
    10.  httpClient.DefaultRequestHeaders.Add("Sec-Ch-Ua-Mobile", "?0");
    11.  httpClient.DefaultRequestHeaders.Add("Sec-Ch-Ua-Platform", "\"Windows\"");
    12.  httpClient.DefaultRequestHeaders.Add("Sec-Fetch-Dest", "document");
    13.  httpClient.DefaultRequestHeaders.Add("Sec-Fetch-Mode", "navigate");
    14.  httpClient.DefaultRequestHeaders.Add("Sec-Fetch-Site", "none");
    15.  httpClient.DefaultRequestHeaders.Add("Sec-Fetch-User", "?1");
    16.  httpClient.DefaultRequestHeaders.Add("Upgrade-Insecure-Requests", "1");
    1.  
    2.  //匹配
    3.  var currencyElement = 
    4.  htmlDocument.DocumentNode.SelectSingleNode("//div[@id='hr_app_cid_1']");
    5.  if (currencyElement != null)
    6.  {
    7.      // 查找包含人民币价格的元素
    8.      var priceElement = currencyElement.SelectSingleNode(".//div[@class='virtual overflow']");
    9.      if (priceElement != null)
    10.      {
    11.       // 提取人民币价格
    12.       var chinesePrice = priceElement.InnerText.Trim();
    13.      }
    14.  
    15.  }

    美元价格也是类似的规则，运行结果

     7.✨获取CSDN热点

    模拟请求，修改规则

    1.   var httpClient = new HttpClient();
    2.   var referer = "xxx";
    3.   httpClient.DefaultRequestHeaders.Add("referer", referer);
    4.   httpClient.DefaultRequestHeaders.Add("User-Agent", GetUserAgent());
    5.   var response = await httpClient.GetAsync("xxx/");
    6.   response.EnsureSuccessStatusCode();
    7.   var responseBody = await response.Content.ReadAsStringAsync();
    8.  
    9.   var contentList = new Liststring, string, string>>();
    10.  
    11.   var htmlDocument = new HtmlDocument();
    12.   htmlDocument.LoadHtml(responseBody);
    13.  
    14.   //  var itemNodes = htmlDocument.DocumentNode.SelectNodes("//div[contains(@class, 'headswiper-item')]");
    15.   var items = htmlDocument.DocumentNode.SelectNodes("//div[@class='headswiper-item']");
    16.  
    17.   if (items != null)
    18.   {
    19.        foreach (var item in items)
    20.        {
    21.            var titleNode = item.SelectSingleNode(".//a[@class='title']");
    22.            var nameNode = item.SelectSingleNode(".//p[@class='name']");
    23.            var linkNode = item.SelectSingleNode(".//a[@class='title']");
    24.  
    25.  
    26.            if (titleNode != null && nameNode != null && linkNode != null)
    27.            {
    28.                var title = titleNode.InnerText.Trim();
    29.                var name = nameNode.InnerText.Trim();
    30.                var link = linkNode.GetAttributeValue("href", "");
    31.                contentList.Add(new Tuple<string, string, string>(title, name, link));
    32.            }
    33.        }
    34.   }
    35.   return View(contentList);

    or 使用正则

    1.  
    2.   // 使用正则表达式匹配所有符合条件的 div 元素内容
    3.   var regex = new Regex(@"
      ", RegexOptions.Singleline);
    4.   var matches = regex.Matches(responseBody);
    5.   foreach (Match match in matches)
    6.   {
    7.       //去除Vuehtml
    8.       string s1temp = match.Groups[1].Value.Trim();
    9.       int i1 = s1temp.IndexOf(">")+1;
    10.       int i2  = s1temp.Length - i1 - 1;
    11.       string  hotstr = s1temp.Substring(i1, i2);
    12.       
    13.       contentList.Add(hotstr);
    14.   }

    运行结果

    8.🎃 获取编程语言排行榜

    1.             var response = await httpClient.GetAsync("xxx");
    2.             response.EnsureSuccessStatusCode();
    3.             var htmlContent = await response.Content.ReadAsStringAsync();
    4.  
    5.             var htmlDocument = new HtmlDocument();
    6.             htmlDocument.LoadHtml(htmlContent);
    7.  
    8.             // 获取表格内容
    9.             var table = htmlDocument.DocumentNode.SelectSingleNode("//table[@class='w-min min-w-full table-fixed divide-y-2 divide-gray-200 text-sm dark:divide-gray-700']");
    10.  
    11.             ViewBag.TableContent = table?.OuterHtml;

    9.🖥️获取小破站热门

    1.  var httpClient = new HttpClient();
    2.  var referer = "https://xxx";
    3.  httpClient.DefaultRequestHeaders.Add("Referer", referer);
    4.  httpClient.DefaultRequestHeaders.Add("User-Agent", GetUserAgent());
    5.  var response = await httpClient.GetAsync("https://xxx");
    6.  response.EnsureSuccessStatusCode();
    7.  var responseBody = await response.Content.ReadAsStringAsync();
    8.  var jsonRes = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(responseBody);
    9.  var tempArr = new List();
    10.  
    11.  int itid = 1;
    12.  foreach (var itemj in jsonRes.data.list)
    13.  {
    14.      string tt = itemj.keyword.ToString();
    15.    //  string hot = itemj.hot_value.ToString();
    16.      tempArr.Add(new MItem
    17.      {
    18.          Index = itid++,
    19.          Title = tt,
    20.          Hot = "",
    21.          Url = "https://search.bilibili.com/all?keyword=" + tt+ "&order=click",
    22.          MobileUrl = "https://search.bilibili.com/all?keyword=" + tt+ "&order=click"
    23.      });
    24.  }
    25.  
    26.  var md = new MData
    27.  {
    28.      Success = true,
    29.      Title = "小破站",
    30.      Subtitle = "热搜榜",
    31.      UpdateTime = DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss"),
    32.      Data = tempArr
    33.  };
    1.   public class MData
    2.   {
    3.       public bool Success { get; set; }
    4.       public string Title { get; set; }
    5.       public string Subtitle { get; set; }
    6.       public string UpdateTime { get; set; }
    7.       public List Data { get; set; }
    8.   }
    1.  public class MItem
    2.     {
    3.         public int Index { get; set; }
    4.         public string Title { get; set; }
    5.         public string Hot { get; set; }
    6.         public string Url { get; set; }
    7.         public string MobileUrl { get; set; }
    8.     }
    9.

    10.⭐获取某音热门 

    1.   var httpClient = new HttpClient();
    2.   var referer = "xxx";
    3.   httpClient.DefaultRequestHeaders.Add("referer", referer);
    4.   httpClient.DefaultRequestHeaders.Add("User-Agent", GetUserAgent());
    5.   var response = await httpClient.GetAsync("xxx");
    6.   response.EnsureSuccessStatusCode();
    7.   var responseBody = await response.Content.ReadAsStringAsync();
    8.   var jsonRes = Newtonsoft.Json.JsonConvert.DeserializeObject<dynamic>(responseBody);
    9.   var tempArr = new List();
    10.  
    11.   int itid = 1;
    12.   foreach (var itemj in jsonRes.word_list)
    13.   {
    14.       string tt = itemj.word.ToString();
    15.       string hot  =itemj.hot_value.ToString();
    16.  
    17.  
    18.       tempArr.Add(new MItem
    19.       {
    20.           Index = itid++,
    21.           Title = tt,
    22.           Hot = hot,
    23.           Url = "https://www.douyin.com/search/" + tt,
    24.           MobileUrl = "https://www.douyin.com/search/" + tt
    25.       }); 
    26.   }
    27.  
    28.   var douyinData = new MData
    29.   {
    30.       Success = true,
    31.       Title = "某音",
    32.       Subtitle = "热搜榜",
    33.       UpdateTime = DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss"),
    34.       Data = tempArr
    35.   };

    注意：如果目标有IP限制、Cookie、签名等，则需编写相应的对策 。

    此外，还可以通过WinFrom WebBowser控件实现加载完DOM再匹配信息实现爬取效果。

    这里只做抛砖引玉，后续是否入狱自行探索，请勿使用此技术做违法的事情！！！💀 

    在使用爬虫技术时，必须遵守相关法律法规，尊重网站经营者的权益，避免对网站或系统造成干扰或破坏。同时，对于涉及公民个人信息的爬虫行为，应格外谨慎，确保不侵犯他人的隐私权。

    END

  • 相关阅读:
    Angular RouterModule.forRoot(ROUTES) 和 forChild(ROUTES)的区别
    微信小程序_20,使用第三方npm包
    HTML+CSS网页设计期末课程大作业 【茶叶文化网站设计题材】web前端开发技术 web课程设计 网页规划与设计
    AVL的代码剖析(c++)
    nginx子域名配置单独的日志文件
    MySQL面试题全解析：准备面试所需的关键知识点和实战经验
    水利水电工程资质怎么办理，水利水电工程施工总承包三级资质办理条件有哪些
    论文速览 | arxiv 2023, 马氏距离感知训练在分布外检测中的应用
    NoSQL之Redis配置与优化
    多姿多彩的编程世界之配色方案
  • 原文地址：https://blog.csdn.net/lmnotlm/article/details/137811018