From 878a2ff1ec5607b5c417a57a9ff0a7daa6e5b48e Mon Sep 17 00:00:00 2001 From: Sumit Gautam Date: Wed, 24 Jun 2026 17:53:51 +0530 Subject: [PATCH] Automatic CF clearance --- README.md | 60 ++++++--- copilot/__init__.py | 3 +- copilot/browser.py | 310 ++++++++++++++++++++++++++++++++++++++++++-- copilot/client.py | 134 ++++++++++++++++--- copilot/driver.py | 56 ++++++-- server/api.py | 23 +++- 6 files changed, 526 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 338744e..b9869df 100644 --- a/README.md +++ b/README.md @@ -73,9 +73,7 @@ playwright install chromium python -m copilot login ``` -The browser **closes by itself** once sign-in is detected β€” you don't need to press Enter or close it manually. After sign-in it sends one short warm-up message to mint the chat token (so a brief "finishing setup…" appears, and a tiny throwaway chat lands in your history). The steps are logged to `session/login.log` if anything goes wrong. That's it: your session is saved under `session/` (git-ignored, never shared) and reused on every run. - -> πŸ’‘ You can even skip step 3: the **first** time you call `chat()` or start the server, it opens the sign-in browser for you automatically. +The browser **closes by itself** once sign-in is detected β€” you don't need to press Enter or close it manually. After sign-in it sends one short warm-up message that mints the chat token **and** passes Cloudflare's "verify you're human" check in the same step (a brief "finishing setup…" appears, and a tiny throwaway chat lands in your history). If a checkbox shows up, click it in that login window. The steps are logged to `session/login.log` if anything goes wrong. That's it: your session is saved under `session/` (git-ignored, never shared) and reused on every run β€” so your first request works right away. --- @@ -83,7 +81,7 @@ The browser **closes by itself** once sign-in is detected β€” you don't need to Prefer a container? You can run the OpenAI-compatible server in Docker once you've signed in. -> **Sign in on the host first.** The login step above opens a *visible* browser, which can't run inside the headless container β€” so run `python -m copilot login` on your host to populate `session/`. The container mounts that folder and only does the automatic (headless) token refresh from then on. +> **Sign in on the host first.** The login step above opens a *visible* browser, which can't run inside the headless container β€” so run `python -m copilot login` on your host to populate `session/`. The container mounts that folder and reuses the Cloudflare clearance earned on the host. It refreshes the chat token headlessly, but it can't earn *fresh* clearance without a visible browser, so when clearance expires (~30 min) it returns a `503` β€” re-run `python -m copilot login` on the host to refresh `session/`. ```bash docker compose up --build @@ -180,6 +178,30 @@ python -m copilot ask "Hello!" # quick one-shot question --- +## Cloudflare clearance (automatic) + +Copilot's chat sits behind Cloudflare. Access needs a `cf_clearance` cookie, +earned by passing a "verify you're human" check in a real browser, and it lasts +about half an hour. The bridge handles this for you: + +- **At sign-in:** `python -m copilot login` earns clearance as part of the same + warm-up that mints your token, so your first request works immediately. If + Cloudflare shows a checkbox, click it in the login window. +- **When it expires:** if a later request hits the gate, the bridge opens a + browser, passes the check (the checkbox is clicked automatically, or you click + it if one appears), and retries the request for you. You'll see a short + `[copilot] clearance: …` progress log, then the answer. + +On a trusted connection the check often passes invisibly with no window at all. A +datacenter/VPN IP is stricter and more likely to show the checkbox; a residential +connection clears most reliably. + +The **server** never opens a window: when clearance expires it returns a `503` +(`type: "clearance_required"`). Re-clear out of band with `python -m copilot +login`, then retry. + +--- + ## Concurrency & stress test The server bridges a **single** signed-in Copilot account, and Copilot's chat @@ -261,7 +283,7 @@ add a few retries yourself. | [copilot/](copilot/) | The core library: `CopilotClient`, auth, browser sign-in, HTTP driver | | [server/](server/) | The FastAPI OpenAI-compatible server | | [examples/](examples/) | Runnable examples for every feature ([examples/README.md](examples/README.md)) | -| [tests/](tests/) | Test scripts: the concurrency stress test ([tests/stress.py](tests/stress.py)) and the diagnostic/captcha-fix tool ([tests/diagnostic.py](tests/diagnostic.py)) | +| [tests/](tests/) | Test scripts: the concurrency stress test ([tests/stress.py](tests/stress.py)) and the diagnostic & report tool ([tests/diagnostic.py](tests/diagnostic.py)) | | [app.py](app.py) | Starts the server | --- @@ -278,24 +300,25 @@ add a few retries yourself. ## Troubleshooting -**Hit an error? Run the diagnostic first β€” it both *fixes* and *logs*.** +Cloudflare clearance is handled automatically (see above), so most "verify you're +human" issues clear themselves. If a request still fails, run the diagnostic β€” it +refreshes the session and writes a shareable report. ```bash -python tests/diagnostic.py # browser capture + captcha fix + report +python tests/diagnostic.py # browser capture + report python tests/diagnostic.py --report-only # headless/VPS: report only, no browser ``` The default run opens your signed-in browser and asks you to send one short -message. That single action does two things: +message. That single action: -- **Fixes captcha:** it drives a *real* browser on the same `session/profile/` - the bridge uses, so passing any "verify you're human" check earns a fresh - `cf_clearance` cookie. When the turn completes the tool snapshots that session - (cookies + token) into `session/token.json`, so the pure-HTTP driver adopts - the clearance immediately. +- **Refreshes clearance:** it drives a *real* browser on the same + `session/profile/` the bridge uses, so passing any "verify you're human" check + earns a fresh `cf_clearance` cookie, then snapshots the session (cookies + + token) into `session/token.json` for the pure-HTTP driver to adopt. - **Captures the protocol** to `session/ws_capture.log`. A clean turn goes `setOptions` β†’ `send` β†’ `appendText…` β†’ `done`; a `{"event":"challenge", - "method":"cloudflare",…}` frame means Cloudflare gated you (now cleared). + "method":"cloudflare",…}` frame means Cloudflare gated the turn. It also writes `session/diagnostic_report.txt` β€” environment, the *shape* of your session (cookie names + token length, never the values), a live chat probe, and @@ -304,11 +327,10 @@ OAuth codes, and emails are redacted before anything is written. Attach `diagnostic_report.txt` to a GitHub issue (skim it first) and the cause is usually obvious. -> On a headless **server/VPS** you can't open the browser, so the captcha fix -> isn't available there β€” pass `--report-only`, then do the clearance step on a -> machine with a display (or route traffic through a residential connection, -> e.g. a home-PC exit node) since datacenter IPs are where Cloudflare withholds -> clearance and you see `RuntimeError: Copilot error: invalid-event`. +> On a headless **server/VPS** you can't open a browser, so clearance can't be +> earned there β€” pass `--report-only`, and do the clearance step on a machine +> with a display (or route traffic through a residential connection, e.g. a +> home-PC exit node), since datacenter IPs are where Cloudflare is strictest. --- diff --git a/copilot/__init__.py b/copilot/__init__.py index 2028877..d0c0d46 100644 --- a/copilot/__init__.py +++ b/copilot/__init__.py @@ -17,12 +17,13 @@ __version__ = '1.0.0' from .auth import load_auth from .browser import BrowserCopilot from .client import ChatReply, CopilotClient -from .driver import Copilot +from .driver import ClearanceRequired, Copilot __all__ = [ 'CopilotClient', 'ChatReply', 'Copilot', + 'ClearanceRequired', 'BrowserCopilot', 'load_auth', ] diff --git a/copilot/browser.py b/copilot/browser.py index 7dc7b25..72af5ee 100644 --- a/copilot/browser.py +++ b/copilot/browser.py @@ -28,6 +28,7 @@ shapes with ``tests/diagnostic.py`` if Microsoft changes them. from __future__ import annotations import json +import sys import time from datetime import datetime, timezone from pathlib import Path @@ -40,6 +41,29 @@ from .auth import DEFAULT_AUTH_FILE, DEFAULT_PROFILE_DIR COPILOT_URL = "https://copilot.microsoft.com/" +# The Cloudflare Turnstile widget renders inside a cross-origin iframe served +# from challenges.cloudflare.com (page-load interstitial *and* the in-chat gate). +# We reach into that frame to click its checkbox β€” see _click_turnstile. +_TURNSTILE_IFRAME = "iframe[src*='challenges.cloudflare.com'], iframe[src*='turnstile']" + +# A stock desktop Chrome UA used in headless mode. Headless Chromium otherwise +# advertises "HeadlessChrome/..." in both the request UA header and +# navigator.userAgent β€” a blatant bot signal to Cloudflare Turnstile, and a UA +# that can mismatch the cf_clearance UA-binding the curl_cffi driver relies on +# (clearance is bound to the UA that earned it). Pinning a normal Chrome UA makes +# the session look ordinary and keeps the earned clearance reusable. Bump the +# version occasionally to stay current. +_STEALTH_UA = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +) + +# Injected into every frame to hide the residual automation tell that survives +# --disable-blink-features=AutomationControlled in some Chromium builds. +_STEALTH_INIT_JS = ( + "Object.defineProperty(navigator, 'webdriver', {get: () => undefined});" +) + # --- in-page JavaScript ----------------------------------------------------- # Discover the Copilot chat MSAL access token from localStorage. The cache holds @@ -141,6 +165,12 @@ class BrowserCopilot: self._captured_chat_token: Optional[str] = None self._captured_identity_type: Optional[str] = None self._ws_listener_installed = False + # Set True once the page's chat socket streams a reply (an ``appendText`` + # frame). This is auto_clear's true success signal: a reply means the + # browser turn passed the Cloudflare gate, so its cookies are worth + # exporting β€” unlike the cf_clearance value, which often stays unchanged + # when the browser replies using clearance it already holds. + self._warmup_replied = False # -- lifecycle ---------------------------------------------------------- @@ -155,13 +185,26 @@ class BrowserCopilot: launch_kwargs = dict( headless=self.headless, args=["--disable-blink-features=AutomationControlled"], + # Drop the "Chrome is being controlled by automated software" + # switch; its presence is a cheap bot tell Turnstile reads. + ignore_default_args=["--enable-automation"], ) + # Hide the "HeadlessChrome" UA only when actually headless; the visible + # window already uses a normal Chrome UA (and that path works today). + if self.headless: + launch_kwargs["user_agent"] = _STEALTH_UA if self.proxy: launch_kwargs["proxy"] = self._parse_proxy(self.proxy) self._context = self._pw.chromium.launch_persistent_context( self.profile_dir, **launch_kwargs, ) + # Mask the residual navigator.webdriver flag for every frame, before + # any page script (incl. Turnstile's) runs. + try: + self._context.add_init_script(_STEALTH_INIT_JS) + except PlaywrightError: + pass self._page = self._context.pages[0] if self._context.pages else self._context.new_page() self._page.set_default_timeout(self.nav_timeout * 1000) self._page.goto(COPILOT_URL, wait_until="domcontentloaded") @@ -271,15 +314,30 @@ class BrowserCopilot: break token = None + before = self._clearance_value() if detected: - print("Signed in β€” finishing setup (sending a warm-up message)...") - log("warming up to mint/capture the chat token") + print("Signed in β€” finishing setup (warm-up + Cloudflare clearance)...") + log("warming up to mint the chat token and earn cf_clearance") try: - token = self.acquire_chat_token(timeout=max(30, int(deadline - time.time()))) + # A single warm-up turn does double duty: the page opens its chat + # socket (the WS listener reads the token off its URL) and, by + # passing the in-chat Cloudflare gate, earns the cf_clearance the + # pure-HTTP driver reuses β€” so the first `ask` after login needs no + # second browser. We click any Turnstile and wait for the reply. + self._warmup_replied = False + if self._send_warmup(): + self._await_gate_pass( + before, timeout=max(30, int(deadline - time.time())) + ) + token = self.access_token() except PlaywrightError as exc: log(f"warm-up error: {exc}") + cleared = self._clearance_value() != before or self._warmup_replied log(f"chat token captured: {'yes' if token else 'no'}" - f" (identity={self._captured_identity_type})") + f" (identity={self._captured_identity_type});" + f" clearance earned: {'yes' if cleared else 'no'}") + print("Cloudflare clearance earned." if cleared + else "Note: clearance not confirmed; first request may open a browser.") else: log(f"not signed in within {timeout}s; snapshotting current state") print("Sign-in not detected; saving whatever session state exists.") @@ -384,13 +442,16 @@ class BrowserCopilot: def on_ws(ws): try: url = ws.url - if "/c/api/chat" not in url or "accessToken=" not in url: + if "/c/api/chat" not in url: return - q = parse_qs(urlparse(url).query) - tok = (q.get("accessToken") or [None])[0] - if tok: - self._captured_chat_token = tok - self._captured_identity_type = (q.get("X-UserIdentityType") or [None])[0] + if "accessToken=" in url: + q = parse_qs(urlparse(url).query) + tok = (q.get("accessToken") or [None])[0] + if tok: + self._captured_chat_token = tok + self._captured_identity_type = (q.get("X-UserIdentityType") or [None])[0] + # Watch reply frames so auto_clear knows the turn passed the gate. + ws.on("framereceived", self._on_chat_frame) except Exception: pass @@ -400,6 +461,20 @@ class BrowserCopilot: except PlaywrightError: pass + def _on_chat_frame(self, payload) -> None: + """Flag a passed turn when the chat socket streams reply content. + + An ``appendText`` (or ``imageGenerated``) frame means the warm-up reply is + flowing, i.e. Cloudflare let the turn through β€” auto_clear's success + signal. A ``challenge`` frame contains neither, so this never false-fires + on the gate itself.""" + try: + data = payload if isinstance(payload, str) else bytes(payload).decode("utf-8", "ignore") + except Exception: + return + if "appendText" in data or "imageGenerated" in data: + self._warmup_replied = True + def _send_warmup(self, text: str = "hi") -> bool: """Send one message through the page composer to mint the chat token. @@ -467,6 +542,221 @@ class BrowserCopilot: self._page.wait_for_timeout(500) return self.access_token() + @staticmethod + def _clear_log(msg: str) -> None: + """Emit an ``auto_clear`` progress line to stderr (keeps stdout clean).""" + print(f"[copilot] clearance: {msg}", file=sys.stderr, flush=True) + + def _clearance_value(self) -> Optional[str]: + """Return the current ``cf_clearance`` cookie value, or ``None``. + + Cloudflare mints a *new* ``cf_clearance`` when a challenge is solved, so a + change in this value is the reliable signal that fresh clearance was + actually earned (:meth:`auto_clear` waits on it). The cookie is set on the + ``.copilot.microsoft.com`` domain.""" + if self._context is None: + return None + try: + for c in self._context.cookies(): + if c.get("name") == "cf_clearance": + return c.get("value") + except PlaywrightError: + pass + return None + + def _click_turnstile(self, timeout_ms: int = 4000) -> bool: + """Best-effort: click the Cloudflare Turnstile checkbox if one is showing. + + Returns True if a checkbox was clicked. The widget lives in a cross-origin + Cloudflare iframe whose checkbox can sit behind nested iframes / shadow + roots, so we try three escalating strategies (the recursive-finder idea + from DrissionPage-based bypassers, adapted to Playwright): + + 1. Scan *all* frames β€” ``page.frames`` is flat and includes frames nested + inside shadow roots that a top-level CSS ``frame_locator`` can't + reach β€” for the Cloudflare challenge frame, and click its checkbox. + Playwright locators pierce open shadow roots inside that frame for us. + 2. Fall back to the top-level ``frame_locator`` selector. + 3. Last resort: click the iframe host element at the checkbox offset + (left-of-centre), where the real checkbox sits. + + A click only *passes* when Cloudflare already trusts this browser; on a + low-trust session (datacenter/VPN IP) it can escalate to a puzzle a click + can't solve β€” :meth:`auto_clear`'s caller detects that (the turn never + replies) and falls back to a visible browser for a human. + """ + if self._page is None: + return False + deadline = time.time() + timeout_ms / 1000 + while True: + # 1. flat-frame scan (robust to shadow-root / nested-iframe nesting) + frame = self._find_turnstile_frame() + if frame is not None and self._click_in_frame(frame): + return True + # 2. top-level frame_locator, then 3. offset click on the host iframe + try: + if self._page.query_selector(_TURNSTILE_IFRAME) is not None: + fl = self._page.frame_locator(_TURNSTILE_IFRAME).first + for sel in ("input[type='checkbox']", "label"): + try: + fl.locator(sel).first.click(timeout=1500) + return True + except PlaywrightError: + continue + if self._click_turnstile_by_offset(): + return True + except PlaywrightError: + pass + if time.time() >= deadline: + return False + self._page.wait_for_timeout(300) + + def _find_turnstile_frame(self): + """Return the Cloudflare challenge frame among all frames, or ``None``. + + ``page.frames`` is a flat list of every frame in the page β€” including ones + embedded inside shadow roots β€” so it finds the Turnstile iframe even when a + top-level CSS selector can't reach it. This is the Playwright equivalent of + the recursive shadow-root/iframe descent the DrissionPage bypassers do.""" + if self._page is None: + return None + try: + for fr in self._page.frames: + u = (fr.url or "").lower() + if "challenges.cloudflare.com" in u or "turnstile" in u: + return fr + except PlaywrightError: + pass + return None + + @staticmethod + def _click_in_frame(frame) -> bool: + """Click the Turnstile checkbox inside an already-resolved challenge frame.""" + for sel in ("input[type='checkbox']", "label", "body"): + try: + frame.locator(sel).first.click(timeout=1500) + return True + except PlaywrightError: + continue + return False + + def _click_turnstile_by_offset(self) -> bool: + """Click the Turnstile iframe host where the checkbox sits (left-of-centre). + + A coordinate click on the host element, used when the checkbox inside the + frame can't be targeted directly (cross-origin isolation / odd markup).""" + try: + host = self._page.query_selector(_TURNSTILE_IFRAME) + if host is None: + return False + box = host.bounding_box() + if not box or box.get("width", 0) < 1: + return False + x = box["x"] + min(30, box["width"] / 2) + y = box["y"] + box["height"] / 2 + self._page.mouse.click(x, y) + return True + except PlaywrightError: + return False + + def _await_gate_pass(self, before_clearance: Optional[str], timeout: int = 60) -> bool: + """Wait for an already-sent warm-up turn to pass the Cloudflare gate. + + Clicks any Turnstile checkbox that appears and returns once the turn + streams a reply (``appendText`` -> gate passed) or a fresh ``cf_clearance`` + is issued, or ``timeout`` elapses. Assumes the caller already installed the + WS listener, reset ``_warmup_replied``, and sent the warm-up. Shared by + :meth:`auto_clear` and :meth:`login` so one warm-up both mints the token + and earns clearance. Returns whether the gate was passed.""" + deadline = time.time() + timeout + clicked = False + while time.time() < deadline: + if self._window_closed(): + self._clear_log("browser window was closed") + break + if self._click_turnstile(timeout_ms=1000) and not clicked: + clicked = True + self._clear_log("clicked the in-chat Turnstile checkbox") + # Success = the turn replied (passed the gate). A changed cf_clearance + # is a secondary signal for the rare case where the cookie refreshes + # but no reply frame is seen. + if self._warmup_replied: + self._clear_log("warm-up reply received β€” gate passed") + break + current = self._clearance_value() + if current and current != before_clearance: + self._clear_log("fresh cf_clearance issued β€” gate passed") + break + self._page.wait_for_timeout(500) + else: + self._clear_log(f"turn did not pass the gate within {timeout}s") + if not self._window_closed(): + self._page.wait_for_timeout(1000) # let the cookie settle to disk + return self._warmup_replied or (self._clearance_value() != before_clearance) + + def auto_clear( + self, path: str = DEFAULT_AUTH_FILE, warmup: bool = True, timeout: int = 60 + ) -> bool: + """Refresh Cloudflare clearance for the pure-HTTP driver, then snapshot it. + + Loads Copilot and clicks any Turnstile checkbox that appears β€” on page + load and, when ``warmup`` and signed in, after sending one throwaway chat + turn (the in-chat Turnstile is the gate observed on the chat socket). Then + snapshots the refreshed cookies + token to ``path`` so the curl_cffi + driver can reuse the earned ``cf_clearance``. + + Headless when constructed with ``headless=True`` (the default): a fully + automatic solve whenever Cloudflare trusts the session. When Cloudflare + escalates to an interactive puzzle (low-trust egress, e.g. a VPN), the + headless click won't pass β€” construct with ``headless=False`` so a human + can finish it. Returns True if a snapshot with cookies was written; the + caller verifies *real* success by retrying the chat turn (a snapshot can + be written even when clearance didn't actually pass). + """ + self._ensure_started() + self._install_ws_listener() + mode = "headless" if self.headless else "visible" + self._clear_log(f"loaded Copilot ({mode}); checking Cloudflare clearance") + + # Remember the pre-existing clearance so we can tell when a *fresh* one is + # earned. The driver only calls us because the current cf_clearance is + # stale, so success = this value changing (or appearing), not merely being + # present. We deliberately do NOT key off the captured chat token: the page + # opens its chat WebSocket (and we capture the token off its URL) *before* + # the Turnstile challenge frame arrives, so that signal fires too early and + # used to close the browser before the checkbox even appeared. + before = self._clearance_value() + + # 1. Solve any challenge gating the page itself on load. + if self._click_turnstile(): + self._clear_log("clicked a page-load Turnstile checkbox") + + # 2. Trigger the in-chat Turnstile (the gate seen on the chat socket) with + # one throwaway turn, then wait for clearance to actually refresh β€” + # clicking any checkbox that appears (headless auto-solve) or letting a + # human click it (visible window). Sending one turn is what the manual + # diagnostic does to earn clearance. + if warmup and self.signed_in(): + self._warmup_replied = False + self._clear_log("sending a warm-up turn to trigger the in-chat challenge") + self._send_warmup() + self._clear_log(f"waiting up to {timeout}s for the turn to pass the gate" + + ("" if self.headless else " (click the checkbox if shown)")) + self._await_gate_pass(before, timeout=timeout) + elif warmup: + self._clear_log("not signed in β€” skipping warm-up; snapshotting state") + + auth = self.export_auth(path=path, stamp=time.time()) + # Report whether the turn actually passed the gate (reply seen or fresh + # clearance), not just that a snapshot was written; the client uses this to + # decide whether to escalate to a visible browser. + earned = bool(auth.get("cookies")) and ( + self._warmup_replied or self._clearance_value() != before + ) + self._clear_log("done β€” clearance refreshed" if earned + else "done β€” no clearance earned") + return earned + def cookies(self) -> Dict[str, str]: """Return the signed-in Microsoft cookies as a name->value dict.""" self._ensure_started() diff --git a/copilot/client.py b/copilot/client.py index ce256cd..f4e1682 100644 --- a/copilot/client.py +++ b/copilot/client.py @@ -23,15 +23,25 @@ anonymous consumer chat is available), or ``proxy=...`` to route through a supported region. """ +import sys import time from dataclasses import dataclass, field from typing import Generator, List, Optional, Union from .auth import AUTH_MAX_AGE, load_auth -from .driver import Copilot +from .driver import ClearanceRequired, Copilot from .models import Conversation, ImageResponse +def _status(msg: str) -> None: + """Emit a recovery/progress line to stderr. + + Goes to stderr (not stdout) so it never mixes into the reply text the CLI and + callers read off stdout. Plain ``print`` keeps it visible by default β€” this is + a personal bridge, not a library that should stay silent.""" + print(f"[copilot] {msg}", file=sys.stderr, flush=True) + + @dataclass class ChatReply: """The full result of a :meth:`CopilotClient.chat` call.""" @@ -76,6 +86,19 @@ class CopilotClient: auth refresh and every request. max_age: Seconds a cached access token is trusted before it is refreshed. + interactive_clear: + When a turn is gated behind a Cloudflare Turnstile (expired + ``cf_clearance``), recover by opening a *visible* browser and refreshing + clearance there (the checkbox is auto-clicked, or a human clicks it), then + retry. Default ``True``. Set ``False`` for headless/server use, where a + :class:`~copilot.driver.ClearanceRequired` error is raised instead of + popping a window. + headless_clear: + Attempt a *headless* clearance refresh before the visible one. Default + ``False``: headless Turnstile solving is unreliable on low-trust egress + (VPN/datacenter IPs) and a failed pass can leave a half-cleared state, so + the dependable path is a visible window. Enable once headless is proven on + your egress. """ def __init__( @@ -83,11 +106,15 @@ class CopilotClient: anonymous: bool = False, proxy: Optional[str] = None, max_age: int = AUTH_MAX_AGE, + interactive_clear: bool = True, + headless_clear: bool = False, ): self._driver = Copilot() self._anonymous = anonymous self._proxy = proxy self._max_age = max_age + self._interactive_clear = interactive_clear + self._headless_clear = headless_clear self._auth: Optional[dict] = None def stream( @@ -101,24 +128,95 @@ class CopilotClient: Starts a new conversation when ``conversation_id`` is ``None``; otherwise continues that conversation. Read ``.conversation_id`` on the returned stream (during/after iteration) to continue the chat later. - """ - auth = self._fresh_auth() - kw = dict( - stream=True, - proxy=self._proxy, - cookies=auth["cookies"] if auth else None, - access_token=auth["access_token"] if auth else None, - identity_type=auth.get("identity_type") if auth else None, - **kwargs, - ) - if conversation_id is None: - # New conversation: have the driver hand back its id. - kw["return_conversation"] = True - else: - kw["conversation_id"] = conversation_id - chunks = self._driver.create_completion(prompt, **kw) - return ChatStream(chunks, conversation_id) + If the turn is gated behind a Cloudflare Turnstile (expired + ``cf_clearance``), it is transparently recovered: clearance is refreshed + in a visible browser (the checkbox is auto-clicked, or a human clicks it) + and the turn is retried. Recovery only happens before any text is emitted, + so output is never duplicated. + """ + return ChatStream( + self._stream_with_recovery(prompt, conversation_id, kwargs), + conversation_id, + ) + + def _stream_with_recovery(self, prompt, conversation_id, kwargs): + """Drive the turn, recovering from an expired-clearance Turnstile by + refreshing clearance in a browser and retrying. + + Recovery opens a *visible* browser by default: headless solving is + unreliable on low-trust egress, so the dependable path is a real window + (auto-clicked checkbox, or a human click). A headless pre-pass runs only + when ``headless_clear`` is set. Recovery happens only before any text is + emitted, so output is never duplicated. + """ + # Browser recovery passes, in order: a headless pre-pass (opt-in) then a + # visible window. Each entry is the ``headless`` flag for that pass. + strategies = [] + if not self._anonymous: + if self._headless_clear: + strategies.append(True) + if self._interactive_clear: + strategies.append(False) + total = len(strategies) + 1 # +1 for the initial as-is attempt + + for attempt in range(total): + auth = self._fresh_auth() + kw = dict( + stream=True, + proxy=self._proxy, + cookies=auth["cookies"] if auth else None, + access_token=auth["access_token"] if auth else None, + identity_type=auth.get("identity_type") if auth else None, + **kwargs, + ) + if conversation_id is None: + kw["return_conversation"] = True # have the driver hand back its id + else: + kw["conversation_id"] = conversation_id + + if attempt: + _status(f"Retrying the message (attempt {attempt + 1}/{total})...") + produced = False # any user-visible output yet? (Conversation doesn't count) + try: + for item in self._driver.create_completion(prompt, **kw): + if not isinstance(item, Conversation): + produced = True + yield item + return + except ClearanceRequired: + if produced: + _status("Cloudflare clearance expired mid-reply β€” can't recover " + "without duplicating output; surfacing the error.") + raise + if attempt >= len(strategies): + # No (more) recovery passes available β€” anonymous, server + # (no visible browser), or the last pass already ran. + _status("Cloudflare clearance could not be refreshed; giving up.") + raise + self._refresh_clearance(headless=strategies[attempt]) + self._auth = None # force a reload of the freshly-snapshotted auth + + def _refresh_clearance(self, headless: bool) -> None: + """Refresh Cloudflare clearance via a browser, re-snapshotting token.json. + + Headless first (automatic when Cloudflare trusts the session); a visible + window second so a human can pass an escalated interactive checkbox. + """ + from .browser import BrowserCopilot + + if headless: + _status("Cloudflare clearance expired β€” attempting a headless refresh...") + else: + _status("Cloudflare clearance expired β€” opening a browser. " + "Click the 'verify you're human' checkbox if it appears.") + bot = BrowserCopilot(headless=headless, proxy=self._proxy) + try: + earned = bot.auto_clear() + finally: + bot.close() + _status("Clearance refreshed." if earned + else "Browser did not earn fresh clearance.") def chat( self, diff --git a/copilot/driver.py b/copilot/driver.py index 2dd9e69..6429899 100644 --- a/copilot/driver.py +++ b/copilot/driver.py @@ -28,6 +28,20 @@ from .protocol import CHAT_WEBSOCKET_URL, CONSENTS_FRAME, SET_OPTIONS_FRAME from .utils import drain_json, is_accepted_format, raise_for_status, to_bytes +class ClearanceRequired(RuntimeError): + """The chat socket demanded a Cloudflare Turnstile token we can't mint here. + + Copilot gates a turn behind a ``challenge`` frame with ``method`` either + ``null`` or ``"cloudflare"`` whenever the session's ``cf_clearance`` cookie is + stale or missing (confirmed by capturing the real web client: it answers a + ``{method:null}`` frame with a ``method:"cloudflare"`` Turnstile token). A + Turnstile token can only be produced by executing Cloudflare's challenge JS in + a real browser, so the pure-HTTP driver can't satisfy it. The caller should + refresh clearance in a browser (see + :meth:`copilot.browser.BrowserCopilot.auto_clear`) and retry the turn. + """ + + class Copilot(AbstractProvider): label = "Microsoft Copilot" url = "https://copilot.microsoft.com" @@ -184,11 +198,27 @@ class Copilot(AbstractProvider): for msg in messages: last_msg = msg event = msg.get("event") - if event == "challenge" and not answered: + if event == "challenge": + method = msg.get("method") + # A Cloudflare Turnstile (method null/"cloudflare") can arrive + # at any point β€” including *after* a proof-of-work challenge was + # already answered this turn β€” and we can never mint its token + # here. Surface it regardless of ``answered`` so a stale + # cf_clearance becomes a clean ClearanceRequired instead of a + # silent 60s idle timeout (the frame would otherwise be ignored). + if method in (None, "cloudflare"): + raise ClearanceRequired( + "Copilot chat is gated behind a Cloudflare Turnstile " + f"(challenge method={method!r}); cf_clearance is stale " + "or missing. Refresh clearance in a browser " + "(copilot.browser.BrowserCopilot.auto_clear) and retry." + ) + if answered: + continue # already answered the PoW for this turn; ignore echo token = self._solve_challenge(msg) if token is None: raise RuntimeError( - f"Unsolvable Copilot challenge (method={msg.get('method')!r}). " + f"Unsolvable Copilot challenge (method={method!r}). " "Microsoft may have escalated to a browser-only challenge; " "fall back to copilot.browser.BrowserCopilot." ) @@ -255,20 +285,24 @@ class Copilot(AbstractProvider): def _solve_challenge(msg: dict): """Return the challenge-response token, or ``None`` if we can't solve it. - Copilot's chat socket precedes the answer with a challenge frame that the - client must acknowledge. An *empty* challenge (no ``method``/``parameter``) - only needs an acknowledging response, so we return an empty token; the - proof-of-work variants are computed in :mod:`copilot.challenges`. A - ``None`` return means the challenge needs a browser-solved token (e.g. a - Cloudflare Turnstile) and the caller should surface that. + Copilot's chat socket precedes the answer with a challenge frame. The + proof-of-work variants (``hashcash``, ``copilot``) are computed in-process + (:mod:`copilot.challenges`). A ``None`` return means the challenge needs a + browser-solved token and the caller must surface that. + + An *empty* challenge (``method``/``parameter`` both null) is NOT a no-op: + capturing the real web client showed it answers ``{method:null}`` with a + ``method:"cloudflare"`` Turnstile token. It only appears when + ``cf_clearance`` is stale, and curl_cffi can't mint a Turnstile token β€” so + we return ``None`` (the caller raises :class:`ClearanceRequired`). The old + "ack an empty challenge with an empty token" behaviour was wrong: it made + the socket wait for a token that never came and silently time out. """ method = msg.get("method") parameter = msg.get("parameter") - if not method and not parameter: - return "" # empty/no-op challenge: just acknowledge it if method == "hashcash" and parameter: return solve_hashcash(parameter) if method == "copilot" and parameter: return solve_copilot_challenge(parameter) - # 'cloudflare' (Turnstile) / unknown PoW needs a browser-solved token. + # method:null / 'cloudflare' (Turnstile) / unknown PoW: browser-only token. return None diff --git a/server/api.py b/server/api.py index 9f4b528..82748a2 100644 --- a/server/api.py +++ b/server/api.py @@ -7,6 +7,7 @@ from fastapi import FastAPI from fastapi.responses import JSONResponse, StreamingResponse from copilot import CopilotClient +from copilot.driver import ClearanceRequired from .config import MODEL_NAME, RATE_LIMIT_BURST, RATE_LIMIT_RPM from .openai_format import ( @@ -20,7 +21,18 @@ from .ratelimit import TokenBucket from .schemas import ChatCompletionRequest app = FastAPI(title="Copilot OpenAI-compatible API", version="1.0.0") -client = CopilotClient() +# Server runs headless and must never pop a visible browser mid-request. With +# both recovery passes disabled, an expired clearance surfaces immediately as a +# 503 (see ClearanceRequired handling below) so an operator can re-clear out of +# band (`python -m copilot login`). Headless auto-solve is intentionally off: +# it's unreliable on low-trust egress and a failed pass can wedge the session. +client = CopilotClient(interactive_clear=False, headless_clear=False) + +_CLEARANCE_HELP = ( + "Cloudflare clearance expired and could not be refreshed headlessly. " + "Re-clear in a browser: run `python -m copilot login` (or `python tests/diagnostic.py`) " + "and pass the 'verify you're human' check, then retry." +) # Self-imposed rate limit on top of the concurrency lock below: this caps # requests-per-minute, the lock caps requests-in-flight. See server/ratelimit.py. @@ -77,6 +89,10 @@ def _stream(prompt: str, model: str, conversation_id=None): conversation_id=stream.conversation_id, ) ) + except ClearanceRequired: + yield sse_event( + stream_chunk(cid, created, model, {"content": f"\n[error: {_CLEARANCE_HELP}]"}, finish="error") + ) except Exception as exc: # surface errors to the client instead of hanging yield sse_event( stream_chunk(cid, created, model, {"content": f"\n[error: {exc}]"}, finish="error") @@ -118,6 +134,11 @@ def chat_completions(req: ChatCompletionRequest): try: with _upstream_lock: # serialize: one upstream chat at a time reply = client.chat(prompt, conversation_id=req.conversation_id) + except ClearanceRequired: + return JSONResponse( + status_code=503, + content={"error": {"message": _CLEARANCE_HELP, "type": "clearance_required"}}, + ) except Exception as exc: return JSONResponse( status_code=502,