Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix invisible iframe scraping #1723

Merged
merged 1 commit into from
Feb 5, 2025
Merged

fix invisible iframe scraping #1723

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 27 additions & 24 deletions skyvern/forge/agent_functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ async def _convert_svg_to_string(


async def _convert_css_shape_to_string(
frame: Page | Frame,
skyvern_frame: SkyvernFrame,
element: Dict,
task: Task | None = None,
step: Step | None = None,
Expand Down Expand Up @@ -279,28 +279,18 @@ async def _convert_css_shape_to_string(
if css_shape:
LOG.debug("CSS shape loaded from cache", element_id=element_id, key=shape_key, shape=css_shape)
else:
# FIXME: support element in iframe
locater = frame.locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')
if await locater.count() == 0:
LOG.info(
"No locater found to convert css shape",
task_id=task_id,
step_id=step_id,
element_id=element_id,
)
return None

if await locater.count() > 1:
LOG.info(
"multiple locaters found to convert css shape",
task_id=task_id,
step_id=step_id,
element_id=element_id,
)
return None

try:
LOG.debug("call LLM to convert css shape to string shape", element_id=element_id)
locater = skyvern_frame.get_frame().locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')
if await locater.count() == 0:
LOG.info(
"No locater found to convert css shape",
task_id=task_id,
step_id=step_id,
element_id=element_id,
key=shape_key,
)
return None

if not await locater.is_visible(timeout=settings.BROWSER_ACTION_TIMEOUT_MS):
LOG.info(
"element is not visible on the page, going to abort conversion",
Expand All @@ -309,9 +299,22 @@ async def _convert_css_shape_to_string(
element_id=element_id,
key=shape_key,
)

skyvern_element = SkyvernElement(locator=locater, frame=skyvern_frame.get_frame(), static_element=element)

_, blocked = await skyvern_frame.get_blocking_element_id(await skyvern_element.get_element_handler())
if blocked:
LOG.info(
"element is blocked by another element, going to abort conversion",
task_id=task_id,
step_id=step_id,
element_id=element_id,
key=shape_key,
)
return None

screenshot = await locater.screenshot(timeout=settings.BROWSER_SCREENSHOT_TIMEOUT_MS)
LOG.debug("call LLM to convert css shape to string shape", element_id=element_id)
screenshot = await locater.screenshot(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)
prompt = prompt_engine.load_prompt("css-shape-convert")

# TODO: we don't retry the css shape conversion today
Expand Down Expand Up @@ -466,7 +469,7 @@ async def cleanup_element_tree_func(frame: Page | Frame, url: str, element_tree:

if _should_css_shape_convert(element=queue_ele):
await _convert_css_shape_to_string(
frame=frame,
skyvern_frame=skyvern_frame,
element=queue_ele,
task=task,
step=step,
Expand Down
12 changes: 5 additions & 7 deletions skyvern/webeye/scraper/scraper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,19 +470,17 @@ async def get_interactable_element_tree_in_frame(

try:
frame_element = await frame.frame_element()
# it will get stuck when we `frame.evaluate()` on an invisible iframe
if not await frame_element.is_visible():
continue
unique_id = await frame_element.get_attribute("unique_id")
except Exception:
LOG.warning(
"Unable to get frame_element",
"Unable to get unique_id from frame_element",
exc_info=True,
)
continue

# it will get stuck when we `frame.evaluate()` on an invisible iframe
if not await frame_element.is_visible():
continue

unique_id = await frame_element.get_attribute("unique_id")

frame_js_script = f"() => buildTreeFromBody('{unique_id}')"

await SkyvernFrame.evaluate(frame=frame, expression=JS_FUNCTION_DEFS)
Expand Down