PHP爬虫类的并发与多线程处理技巧

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

class Crawler {

    private $startUrl;

    public function __construct($startUrl) {

        $this->startUrl = $startUrl;

    }

    public function crawl() {

        // 获取初始页面的内容

        $content = $this->getContent($this->startUrl);

        // 解析页面内容，获取需要的信息

        $data = $this->parseContent($content);

        // 处理获取到的信息，进行业务逻辑处理或存储

        $this->processData($data);

        // 获取页面中的链接，并递归抓取

        $urls = $this->getUrls($content);

        foreach ($urls as $url) {

            $content = $this->getContent($url);

            $data = $this->parseContent($content);

            $this->processData($data);

        }

    }

    private function getContent($url) {

        // 发起HTTP请求，获取页面内容

        // ...

        return $content;

    }

    private function parseContent($content) {

        // 解析页面内容，提取需要的信息

        // ...

        return $data;

    }

    private function processData($data) {

        // 处理获取到的信息，进行逻辑处理或存储

        // ...

    }

    private function getUrls($content) {

        // 获取页面中的链接

        // ...

        return $urls;

    }

}

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

class ConcurrentCrawler {

    private $urls;

    public function __construct($urls) {

        $this->urls = $urls;

    }

    public function crawl() {

        $workers = [];

        $urlsNum = count($this->urls);

        $maxWorkersNum = 10; // 最大进程数

        for ($i = 0; $i < $maxWorkersNum; $i++) {

            $pid = pcntl_fork();

            if ($pid == -1) {

                die('fork failed');

            } else if ($pid == 0) {

                for ($j = $i; $j < $urlsNum; $j += $maxWorkersNum) {

                    $this->processUrl($this->urls[$j]);

                }

                exit();

            } else {

                $workers[$pid] = true;

            }

        }

        while (count($workers)) {

            $pid = pcntl_wait($status, WUNTRACED);

            if ($status == 0) {

                unset($workers[$pid]);

            } else {

                $workers[$pid] = false;

            }

        }

    }

    private function processUrl($url) {

        // 发起HTTP请求，获取页面内容

        // ...

        // 解析页面内容，获取需要的信息

        // ...

        // 处理获取到的信息，进行逻辑处理或存储

        // ...

    }

}

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

class MultithreadCrawler extends Thread {

    private $url;

    public function __construct($url) {

        $this->url = $url;

    }

    public function run() {

        // 发起HTTP请求，获取页面内容

        // ...

        // 解析页面内容，获取需要的信息

        // ...

        // 处理获取到的信息，进行逻辑处理或存储

        // ...

    }

}

class Executor {

    private $urls;

    public function __construct($urls) {

        $this->urls = $urls;

    }

    public function execute() {

        $threads = [];

        foreach ($this->urls as $url) {

            $thread = new MultithreadCrawler($url);

            $thread->start();

            $threads[] = $thread;

        }

        foreach ($threads as $thread) {

            $thread->join();

        }

    }

}