Spaces:
Sleeping
Sleeping
| import playwright | |
| import playwright.sync_api | |
| from playwright.sync_api import sync_playwright | |
| import json | |
| url = "https://techorama.be/agenda/" | |
| selector_links = "div.m-subject__container-inner.a-box-simple__inner-2 > a" | |
| #TODO | |
| #1 save the structure [{"time":"...", "activities":[{"name_of_activity":"...", "speaker_name": "...", general_topic": "topic name", "room_number":"..". "url":"...", "date":"...", "time":"time"}]} | |
| #go to each link, and fetch the description and add it to the dictionary | |
| #save a csv | |
| data_technorama = [{"time":"", "activities":[{"name_of_activity":"", "speaker_name": "...", "general_topic": "topic name", "room_number":"..", "url":"...", "date":"...", "time":"time"}]}] | |
| selector_schedule_section = "article.o-schedule__section" | |
| selector_time = "h4.o-schedule__section-title" | |
| selector_activity = "article.m-subject" | |
| selector_activity_title = "h3.m-subject__title" | |
| selector_activity_speaker_name = "p.m-subject__name" | |
| selector_activity_room_number = "p.m-subject__room" | |
| selector_activity_track_label = "p.m-subject__track-label" | |
| selector_url = "div.m-subject__container-inner.a-box-simple__inner-2 > a" | |
| def get_text_content_of(element: playwright.sync_api.ElementHandle): | |
| if element is None: return None | |
| return element.text_content() | |
| def get_url_of(element: playwright.sync_api.ElementHandle): | |
| if element is None: return None | |
| return element.get_property("href").json_value() | |
| with sync_playwright() as p: | |
| browser = p.chromium.launch(headless=True) | |
| page = browser.new_page() | |
| page.goto(url) | |
| page.wait_for_timeout(3000) | |
| schedule_section_elems = page.query_selector_all(selector_schedule_section) | |
| sections = [] | |
| for schedule_section_elem in schedule_section_elems: | |
| section_time = schedule_section_elem.query_selector(selector_time).text_content() | |
| section_activities = schedule_section_elem.query_selector_all(selector_activity) | |
| section_activities_list = [] | |
| for section_activity in section_activities: | |
| activity_name = get_text_content_of(section_activity.query_selector(selector_activity_title)) | |
| speaker_name = get_text_content_of(section_activity.query_selector(selector_activity_speaker_name)) | |
| room_number = get_text_content_of(section_activity.query_selector(selector_activity_room_number)) | |
| track_label = get_text_content_of(section_activity.query_selector(selector_activity_track_label)) | |
| url_activity = get_url_of(section_activity.query_selector(selector_url)) | |
| section_activities_list.append({"name_of_activity": activity_name, | |
| "speaker_name": speaker_name, | |
| "room_number":room_number, | |
| "track_label": track_label, | |
| "url": url_activity, | |
| "day_of_week":"Tuesday", | |
| "date": "2024/05/07", | |
| "time":section_time}) | |
| sections.append({ | |
| "time": section_time, | |
| "activities": section_activities_list | |
| }) | |
| page.locator("button:has-text(\"wednesday\")").click() | |
| schedule_section_elems = page.query_selector_all(selector_schedule_section) | |
| for schedule_section_elem in schedule_section_elems: | |
| section_time = schedule_section_elem.query_selector(selector_time).text_content() | |
| section_activities = schedule_section_elem.query_selector_all(selector_activity) | |
| section_activities_list = [] | |
| for section_activity in section_activities: | |
| activity_name = get_text_content_of(section_activity.query_selector(selector_activity_title)) | |
| speaker_name = get_text_content_of(section_activity.query_selector(selector_activity_speaker_name)) | |
| room_number = get_text_content_of(section_activity.query_selector(selector_activity_room_number)) | |
| track_label = get_text_content_of(section_activity.query_selector(selector_activity_track_label)) | |
| url_activity = get_url_of(section_activity.query_selector(selector_url)) | |
| section_activities_list.append({"name_of_activity": activity_name, | |
| "speaker_name": speaker_name, | |
| "room_number":room_number, | |
| "track_label": track_label, | |
| "url": url_activity, | |
| "day_of_week":"Wednesday", | |
| "date": "2024/05/08", | |
| "time":section_time}) | |
| sections.append({ | |
| "time": section_time, | |
| "activities": section_activities_list | |
| }) | |
| # Save sections to JSON file | |
| with open("sections.json", "w") as f: | |
| json.dump(sections, f) | |
| browser.close() | |