【更新】囚生CYのMemo（20231118~）

---

20231118

• 睡了个超级大懒觉，日常八点醒，但天气转冷并不想早起（其实这周四周五都差点迟到，都是因为天冷赖床，本来每天赶8点53分的车，还有一班8点56分兜底，周四连后者都错过了，结果9点30分30秒险些没打到卡，周五是因为在18号线上发消息坐过站到迎春路，当时慌得一批，想想还是只能回头转9号线，好在9点29分多打到卡），吃点东西回去睡回笼觉，结果第二次醒来已是十二点，下午还要陪嘉伟拉长距离，赶紧收拾东西去学校。
• 阳光明媚，6~12℃，最喜欢冬天这样的天气，感觉有用不完的力量。陪嘉伟做上马前最后一次30km拉练，我只跟随了前15km，配速4’44"，心率155bpm，对我来说是很满意的数据，因为周四蛙跳没热身，大腿酸痛得厉害，并不想太苛求自己跑更多。事实上现在我压住75%的最大心率就可以达到4’45"左右的配速，远远优于之前的水平，跑得非常轻松，12km后依然可以轻松聊天，如果不是因为跑前已处战损状态，这样的天气我能一路跑到天黑。
• 不熟悉马拉松的人可能不知道，半马和全马是两种完全不同的运动，前者对于我们这样的业余跑者是可以一口气不补给不停歇地冲下来，与普通跑个10km并没有太大区别，无非是配速慢10~15秒。但是全马过了30km，任何人，哪怕是最顶尖的职业运动员，都会极其痛苦，因为30km之后所有人体内储存的糖原都消耗殆尽，能量补充大多来源于脂肪，但此时距离终点还有10km以上的路程，这是最令人绝望的一段路程。都说马拉松从30km才刚刚开始，绝大多数业余跑者30km之后配速会显著下降。如果这段遇到长上坡、过大桥，很可能就会转为走走停停，因为意志一旦被冲垮，就很难重建。
• 我也认为全马是反人类的项目，但是终有一天我也一定会完赛全马。后AK时代，嘉伟是第一个将要参加全马的学生，首马即上马，壮志凌云，剑指破三，当共勉之。

m3e-base的中文嵌入效果确实是非常好，最近直接拿来不作任何微调，就可以把很脏的招聘信息title直接匹配到职业大典里的标准职业字段，精确度非常高，远远超过之前用的许多基于规则以及应用序列距离的方法。另外还有两个近期的中文预训练模型也很不错，bge和stella，但是模型比m3e大很多，m3e基本上只有半个标准BERT的大小，CPU上也能跑得很快。

最后，该死的textarea问题终于解决：

# -*- coding: utf-8 -*-
# @author: caoyang
# @email: caoyang@163.sufe.edu.cn

import os
import re
import time
import logging
import requests

from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions

class BaseCrawler:
	tag_regex = re.compile(r"<[^>]+>|\n|\t")
	global_timeout = 60
	global_interval = 300
	chrome_user_data_path = r"C:\Users\caoyang\AppData\Local\Google\Chrome\User Data"
	# Convert request headers copied from Firefox to dictionary
	@classmethod
	def headers_to_dict(cls, headers: str) -> dict:
		lines = headers.splitlines()
		headers_dict = {}
		for line in lines:
			key, value = line.strip().split(':', 1)
			headers_dict[key.strip()] = value.strip()
		return headers_dict

	# Easy use of WebDriverWait
	@classmethod
	def check_element_by_xpath(cls, driver, xpath, timeout=30):
		WebDriverWait(driver, timeout).until(lambda _driver: _driver.find_element_by_xpath(xpath).is_displayed())
	
	# @param method: e.g. GET, POST
	def easy_requests(self, method, url, **kwargs):
		while True:
			try:
				response = requests.request(method, url, **kwargs)
				break
			except Exception as e:
				logging.warning(f"Error {method} {url}, exception information: {e}")
				logging.warning(f"Wait for {self.global_interval} seconds ...")
				time.sleep(self.global_interval)
		return response

	# Initialize driver
	def initialize_driver(self, browser="chrome", headless=True, timeout=60, **kwargs):
		browser = browser.lower()
		assert browser in ["chrome", "firefox"], f"Unknown browser name: {browser}"
		return eval(f"self._initialize_{browser}_driver")(headless, timeout, **kwargs)
	
	# Initialize Google Chrome driver
	def _initialize_chrome_driver(self, headless, timeout, **kwargs):
		chrome_options = webdriver.ChromeOptions()		
		chrome_options.add_argument(f"user-data-dir={self.chrome_user_data_path}")	# Import user data
		if headless:
			chrome_options.add_argument("--headless")
		driver = webdriver.Chrome(chrome_options=chrome_options)
		driver.set_page_load_timeout(timeout)
		if not headless:
			driver.maximize_window()
		return driver

	# Initialize Mozilla Firefox driver
	def _initialize_firefox_driver(self, headless, timeout, **kwargs):
		options = webdriver.FirefoxOptions()
		if headless:
			options.add_argument("--headless")
		driver = webdriver.Firefox(options=options)
		driver.set_page_load_timeout(timeout)
		if not headless:
			driver.maximize_window()
		return driver

	# Get cookies by driver
	def get_cookies(self, url, driver=None, browser="chrome"):
		quit_flag = False
		if driver is None:
			# If there is no driver passed
			quit_flag = True
			driver = self.initialize_driver(browser=browser, headless=True, timeout=30)
		driver.get(url)
		cookies = driver.get_cookies()
		def _cookie_to_string(_cookies):
			_string = str()
			for _cookie in _cookies:
				_name = _cookie["name"]
				_value = _cookie["value"].replace(' ', '%20') # %20 refers to space char in HTML
				_string += f"{_name}={_value};"
			return _string.strip()
		if quit_flag:
			driver.quit()
		return _cookie_to_string(cookies)


class ChatGLMCrawler(BaseCrawler):
	urls = {"home": "https://chatglm.cn/main/detail",	# Home URL
			}
	layout_xpaths = {"input-box"			: "//textarea[@class=\"scroll-display-none\"]",					# XPath of the input box for human
					 # "input-box"			: "//div[@class=\"input-box-inner\"]",							# XPath of the input box for human (div tag cannot be interacted)
					 "send-button-1"		: "//img[@class=\"enter_icon\"]",								# XPath of the button to send text of input box
					 "send-button-2"		: "//div[@class=\"enter\"]",									# XPath of the button to send text of input box
					 "chat-area"			: "//div[@id=\"session-container\"]",							# XPath of the chat area which contains all the talks (consist of several chat boxes)
					 "human-box"			: "//div[@class=\"pr\"]",										# XPath of human chat box
					 "ai-box"				: "//div[@class=\"answer-content-wrap\"]",						# XPath of AI chat box
					 "ai-box-text"			: "//div[@class=\"markdown-body\"]",							# XPath of the text contained in AI chat box
					 "create-new-button"	: "//div[@class=\"new-session-button\"]",						# XPath of create new talk
					 "like-or-dislike-area"	: "//div[@class=\"interact-operate\"]",							# XPath of div tag contains like and dislike icons
					 "delete-session"		: "//span[@class=\"btn delete\"]",								# XPath of button to delete old talk
					 }
	forbidden_strings = []
	def __init__(self):
		super(ChatGLMCrawler, self).__init__()

	# @param driver		: WebDriver object
	# @param prompt		: The question you would like to ask AI
	# @param model_name	: One of the key in `model_card_xpath`, e.g. "chatgpt3.5(16k)"
	def request(self, driver, prompt, first_trial=True):
		prompt = prompt.replace('\n', "\\n")
		if first_trial:
			driver.get(self.urls["home"])
		self.check_element_by_xpath(driver, xpath=self.layout_xpaths["input-box"], timeout=60)		# Check if input box is rendered
		self.check_element_by_xpath(driver, xpath=self.layout_xpaths["send-button-1"], timeout=60)	# Check if send button is rendered
		self.check_element_by_xpath(driver, xpath=self.layout_xpaths["send-button-2"], timeout=60)	# Check if send button is rendered
		# Delete old talk
		try:
			driver.find_element_by_xpath(self.layout_xpaths["delete-session"]).click()
			logging.info("Delete old talk ...")
		except:
			logging.info("No old talk found ...")
		# Request
		logging.info("Prompt ...")
		try:
			## Method 1: Use `element.send_keys`
			driver.find_element_by_xpath(self.layout_xpaths["input-box"]).send_keys(prompt)	# Input the given prompt
			logging.info("  - ok!")
		except:
			# ## Method 2: Use Javascript with one auguments (Fail)
			# js = """var txt = arguments[0]; document.getElementsByTagName("textarea")[0].value = txt;"""		
			# driver.execute_script(js, prompt)
			# logging.info("  - Use Javascript to input ...")

			# # Method 3: Use Javascript with event dispatch (Fail)
			# js = """var elm = arguments[0], txt = arguments[1]; elm.value += txt; elm.dispatchEvent(new Event('change'));"""
			# element = driver.find_element_by_xpath(self.layout_xpaths["input-box"])
			# driver.execute_script(js, element, prompt)
			# logging.info("  - Use Javascript to input ...")

			# # Method 4: Use keyboard operation (Success)
			# import pyperclip
			# pyperclip.copy(prompt)
			# time.sleep(1)
			# driver.find_element_by_xpath(self.layout_xpaths["input-box"]).send_keys(Keys.CONTROL, 'v')
			# logging.info("  - Use keyboard to input ...")

			# Method 5: Use Javascript with DispatchEvent (Success)
			js = """var txt = arguments[0];
			const textarea = $("textarea");
			var nativeTextAreaValueSetter = Object.getOwnPropertyDescriptor(window.HTMLTextAreaElement.prototype, "value").set;
			nativeTextAreaValueSetter.call(textarea, txt);
			const event = new Event("input", {bubbles: true});
			textarea.dispatchEvent(event);"""
			driver.execute_script(js, prompt)
			logging.info("  - Use Javascript to input ...")

		while True:
			# The button is dynamic and sometimes fail to click on
			try:
				driver.find_element_by_xpath(self.layout_xpaths["send-button-1"]).click()		# Click on the button to send the prompt
				logging.info("Use send button 1 ...")
				break
			except:
				try:
					driver.find_element_by_xpath(self.layout_xpaths["send-button-2"]).click()	# Click on the button to send the prompt
					logging.info("Use send button 2 ...")
					break
				except:
					logging.info("Use send button error ...")
					raise Exception("Use send button error ...")
		# Wait for response
		self.check_element_by_xpath(driver, xpath=self.layout_xpaths["chat-area"], timeout=30)	# Check if chat area is rendered
		self.check_element_by_xpath(driver, xpath=self.layout_xpaths["human-box"], timeout=30)	# Check if human chat box is rendered
		self.check_element_by_xpath(driver, xpath=self.layout_xpaths["ai-box"], timeout=30)		# Check if AI chat box is rendered
		finish_flag = True	# Indicating if AI generation is finished
		while finish_flag:
			try:
				# If like or dislike appear, then stop
				driver.find_element_by_xpath(self.layout_xpaths["like-or-dislike-area"])
				finish_flag = False
			except:
				ai_box_text = driver.find_element_by_xpath(self.layout_xpaths["ai-box-text"])						# Find AI response text element
				# ai_box_text = driver.find_element_by_xpath(self.layout_xpaths["ai-box"])							# Find AI response text element
				ai_box_text_inner_html = ai_box_text.get_attribute("innerHTML")										# Get inner HTML of the element
				response = self.tag_regex.sub(str(), ai_box_text_inner_html).strip("\n\t ").replace('\n', '\\n')	# Process response text
				forbidden_flags = [forbidden_string in response for forbidden_string in self.forbidden_strings]
				if sum(forbidden_flags) > 0:
					# It indicates that a forbidden string occurs
					finish_flag = False
		# Extract AI response text
		ai_box_text = driver.find_element_by_xpath(self.layout_xpaths["ai-box-text"])	# Find AI response text element
		ai_box_text_inner_html = ai_box_text.get_attribute("innerHTML")					# Get inner HTML of the element
		response = self.tag_regex.sub(str(), ai_box_text_inner_html).strip("\n\t ")		# Process response text
		return response

	# @param data_path: EXCEL file of job descriptions
	# @param save_path: file path for storing AI response
	# @param model_name: defined in model_card_xpaths
	def demo(self, model_name="chatgpt3.5(16k)"):
		driver = self.initialize_driver(browser="chrome", headless=False, timeout=60)
		driver.implicitly_wait(15) # 
		prompt = "给我讲述一下《基督山伯爵》的故事，500字左右。"
		response = self.request(driver, prompt)
		with open(f"d:/answer-chatglm.txt", 'w', encoding="utf8") as f:
			f.write(response)
		time.sleep(5)
		driver.quit()

if __name__ == "__main__":
	crawler = ChatGLMCrawler()
	crawler.demo()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222