Harder ignoring of ui prompts
This commit is contained in:
@@ -100,6 +100,13 @@ class InstagramExtractor:
|
|||||||
"allow the use of cookies",
|
"allow the use of cookies",
|
||||||
"use of cookies",
|
"use of cookies",
|
||||||
"cookies and similar technologies",
|
"cookies and similar technologies",
|
||||||
|
"cookies policy",
|
||||||
|
"cookie preferences",
|
||||||
|
"learn more about cookies",
|
||||||
|
"review or change your choices",
|
||||||
|
"essential cookies",
|
||||||
|
"optional cookies",
|
||||||
|
"cookies from other companies",
|
||||||
"meta products",
|
"meta products",
|
||||||
"safer experience",
|
"safer experience",
|
||||||
"information we receive from cookies",
|
"information we receive from cookies",
|
||||||
@@ -163,7 +170,9 @@ class InstagramExtractor:
|
|||||||
if buffer:
|
if buffer:
|
||||||
block = "\n".join(buffer)
|
block = "\n".join(buffer)
|
||||||
if not _looks_like_language_list(block):
|
if not _looks_like_language_list(block):
|
||||||
cleaned_lines.extend(buffer)
|
cleaned_lines.extend(
|
||||||
|
[line for line in buffer if not _looks_like_ui_prompt(line)]
|
||||||
|
)
|
||||||
buffer.clear()
|
buffer.clear()
|
||||||
|
|
||||||
for line in lines:
|
for line in lines:
|
||||||
@@ -171,6 +180,8 @@ class InstagramExtractor:
|
|||||||
if not stripped:
|
if not stripped:
|
||||||
flush_buffer()
|
flush_buffer()
|
||||||
continue
|
continue
|
||||||
|
if _looks_like_ui_prompt(stripped):
|
||||||
|
continue
|
||||||
if len(stripped) <= 24:
|
if len(stripped) <= 24:
|
||||||
buffer.append(stripped)
|
buffer.append(stripped)
|
||||||
else:
|
else:
|
||||||
|
|||||||
Reference in New Issue
Block a user