Merge branch 'send-words' into data-sources

src/aihawk/apply.py

@@ -140,20 +140,29 @@ def verify_system(guidance: str) -> str:
         return VERIFY_SYSTEM
     return f"{VERIFY_SYSTEM}\n\nAbout this site. Where this differs from the rules above, the rules above win:\n\n{guidance}"
 
-# What a control says when it sends a form, in the languages forms most often come in.
+# What a control says when it sends a form, in the languages forms most often come in. The stems take the verb's other
+# forms: Candidatar-me, candidarsi, inviare, envoyez.
 _SENDS = re.compile(
-    r"\b(submit|send|invia|inviar|candidati|candidatura|bewerben|absenden|senden|abschicken|envoyer|postuler|soumettre|enviar|solicitar|aplicar"
-    r"|verzenden|versturen|indienen|solliciteren|skicka|ansök|ansøg|sende|lähetä|wyślij|aplikuj|odeslat|odoslať|küldés|trimite|gönder|başvur"
-    r"|confirm|finish|complete|conferma|confermare|completa|completare|termina|confirmar|completar|finalizar|terminar|bestätigen|abschließen"
-    r"|fertigstellen|beenden|confirmer|terminer|finaliser|bevestigen|voltooien|bekräfta|slutför|bekreft|bekræft|fullfør|fuldfør|potwierdź|zakończ"
-    r"|potvrdit|dokončit|confirmă|finalizează|onayla|tamamla|подтвердить|завершить|отправить|откликнуться|إرسال)\b"
+    r"\b(submit|send|invia|inviar\w*|candidati|candidatevi|candid(?:at)?ar\w*|candidatura|bewerben|absenden|senden|abschicken|versenden|einreichen"
+    r"|envoyer|envoyez|post[uú]l\w*|soumett\w*|enviar|envíe|envie|solicitar|solicite|submet\w*|aplicar"
+    r"|verzenden|versturen|verstuur|indienen|solliciteren|solliciteer\w*|skicka|ansök|ansøg|sende|lähetä|wyślij|aplikuj|odeslat|odoslať|küldés|trimite|gönder"
+    r"|başvurun|başvur|başvuru\s+yap|подать|подайте|отправить|откликнуться|إرسال|تقديم)\b"
     # These scripts write no space between words, so a word has no edge to look for: 送信する is still 送信.
-    r"|送信|応募|发送|提交|申请|제출|보내기|確認|完了|确认|完成|확인|완료",
+    r"|送信|応募|申し込み|提出|发送|提交|投递|申请|제출|보내기|지원하기",
     re.IGNORECASE,
 )
+# Confirm, Finish and Complete, in the languages above. They send only as the whole label, or with a word for an
+# application or a submission after them. About something else, such as an email address or a profile, they are a step.
+_FINISH_LATIN = r"(?:confirm|finish|complete|conferma|confermare|completa|completare|termina|finalizza|finalizzare|confirmar|completar|finalizar|terminar|bestätigen|abschließen|fertigstellen|beenden|confirmer|terminer|finaliser|bevestigen|voltooien|bekräfta|slutför|bekreft|bekræft|fullfør|fuldfør|potwierdź|zakończ|potvrdit|dokončit|confirmă|finalizează|onayla|tamamla|подтвердить|завершить)"
+_FINISH_CJK = r"(?:確認|完了|确认|完成|확인|완료)"
+_APPLICATION = r"(?:application\w*|submission\w*|candidatur\w*|bewerbung\w*|domanda|solicitud\w*|solicitação|sollicitatie|ansökan|ansøgning|aplikacja|başvuru|заявк\w*|申请|応募|지원)"
+_FINISH_ANY = re.compile(rf"\b{_FINISH_LATIN}\b|{_FINISH_CJK}", re.IGNORECASE)
+_FINISH_WHOLE = re.compile(rf"\W*(?:{_FINISH_LATIN}|{_FINISH_CJK})\W*", re.IGNORECASE)
+_FINISH_WITH_APPLICATION = re.compile(rf"\b{_FINISH_LATIN}\b(?:\W+\w+){{0,2}}?\W+{_APPLICATION}", re.IGNORECASE)
 _APPLY = re.compile(r"\b(apply|applica)\b", re.IGNORECASE)
 # What a button inside a form says when it does not send it: a step, an answer (Ashby's Yes and No are buttons with no
-# type), or a file to attach.
+# type), or a file to attach. Continue is a step: a submit-type Continue is the next step of a multi-step form far more
+# often than the send, and refusing it would stop every such form.
 _STEPS = re.compile(
     r"\b(next|continue|back|previous|add|avanti|continua|indietro|aggiungi|weiter|suivant|siguiente"
     r"|yes|no|si|oui|non|ja|nein|upload|attach|browse|carica|choose file)\b",
@@ -188,16 +197,24 @@ def _asks_for_enter(element: dict[str, Any]) -> bool:
     return bool(ENTER_ASKS.search(f"{element.get('placeholder', '')} {element.get('aria_label', '')}"))
 
 
+def _finishes(text: str) -> bool:
+    """Whether Confirm, Finish or Complete sends: as the whole label, or with an application word after it."""
+    return bool(_FINISH_WHOLE.fullmatch(text) or _FINISH_WITH_APPLICATION.search(text))
+
+
 def sends_form(element: dict[str, Any], form_touched: bool) -> bool:
     """Whether clicking this element would send the application. Its words decide first: a send word in one of the languages
-    `_SENDS` has, or Apply once the form is touched or the element is in one. A link is judged by its words alone. Otherwise
-    a submit button, or a button with no type inside a form, sends unless its words name a step, an answer or a file."""
+    `_SENDS` has, Confirm, Finish or Complete as `_finishes` reads it, or Apply once the form is touched or the element is in
+    one. Confirm, Finish or Complete about something else is a step, as Continue is. A link is judged by its words alone.
+    Otherwise a submit button, or a button with no type inside a form, sends unless its words name a step, an answer or a file."""
     tag, kind, text = element.get("tag", ""), element.get("type", ""), element.get("text", "")
-    if _SENDS.search(text):
+    if _SENDS.search(text) or _finishes(text):
         return True
     # "Apply" opens the form from a job's page, and sends it once the form has been filled.
     if _APPLY.search(text) and (form_touched or element.get("in_form")):
         return True
+    if _FINISH_ANY.search(text):
+        return False
     if tag == "a":
         return False
     # A button with no type inside a form is a submit button, as HTML defines it. A step of a long form is one too.

tests/test_apply.py

@@ -74,13 +74,37 @@ def ready(selector: str = "#send") -> list[Any]:
         ({"tag": "iframe", "type": "", "in_form": True, "text": ""}, True, False),
         ({"tag": "iframe", "type": "", "in_form": False, "text": "Widget containing a Cloudflare security challenge"}, True, False),
         ({"tag": "iframe", "type": "", "in_form": True, "text": "reCAPTCHA"}, False, False),
+        # Candidatar-me is a Spanish send from a plain button. A submit-type Continue is the next step of a multi-step form,
+        # and a Continue in a div is one too: neither is held back by its words.
+        ({"tag": "button", "type": "button", "in_form": True, "text": "Candidatar-me"}, True, True),
+        ({"tag": "button", "type": "submit", "in_form": True, "text": "Continue"}, True, False),
+        ({"tag": "div", "type": "", "in_form": False, "text": "Continue"}, True, False),
+        # Confirm, Finish or Complete about an email address or a profile is a step, even as a submit button.
+        ({"tag": "button", "type": "submit", "in_form": True, "text": "Confirm email address"}, True, False),
+        ({"tag": "button", "type": "", "in_form": True, "text": "Complete your profile"}, True, False),
+        ({"tag": "div", "type": "", "in_form": False, "text": "Conferma indirizzo email"}, False, False),
+        ({"tag": "button", "type": "button", "in_form": True, "text": "Completa il profilo"}, True, False),
+        ({"tag": "button", "type": "button", "in_form": True, "text": "Finish setting up"}, True, False),
+        # Confirm, Finish or Complete with an application or a submission after it sends.
+        ({"tag": "button", "type": "button", "in_form": True, "text": "Confirm application"}, True, True),
+        ({"tag": "button", "type": "button", "in_form": True, "text": "Complete your job application"}, True, True),
+        ({"tag": "button", "type": "button", "in_form": True, "text": "Finish submission"}, True, True),
+        ({"tag": "div", "type": "", "in_form": False, "text": "Confirmar candidatura"}, False, True),
     ],
 )
 def test_sends_form(element: dict[str, Any], touched: bool, sends: bool) -> None:
     assert sends_form(element, touched) is sends
 
 
-@pytest.mark.parametrize("word", ["Send", "Senden", "Inviar", "Verzenden", "Versturen", "Skicka", "Wyślij", "Odeslat", "Отправить", "送信", "提交", "제출"])
+@pytest.mark.parametrize(
+    "word",
+    [
+        "Send", "Senden", "Inviar", "Verzenden", "Versturen", "Skicka", "Wyślij", "Odeslat", "Отправить", "送信", "提交", "제출",
+        # The other forms of the verbs: Candidatar-me, Inviare, Envoyez, Soumettez, Einreichen, Versenden.
+        "Candidatar-me", "Candidatarme", "Candidatevi", "Inviare", "Envoyez", "Soumettez", "Einreichen", "Versenden",
+        "Solliciteer", "Submeter", "Başvurun", "Başvuru yap", "Подать", "提出", "投递", "申し込み", "지원하기", "تقديم",
+    ],
+)
 def test_a_control_that_sends_in_another_language_is_held_back_even_with_no_form_around_it(word: str) -> None:
     assert sends_form({"tag": "div", "type": "", "in_form": False, "text": word}, True)