scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/rmadith/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:34: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "#######\n",
    "# Importing the required libraries\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import pandas as pd\n",
    "import advertools as adv\n",
    "\n",
    "import json\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "\n",
    "\n",
    "\n",
    "import google.generativeai as ga\n",
    "import chromadb.utils.embedding_functions as embedding_functions\n",
    "import chromadb\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "id = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-24 18:47:25,925 | INFO | posthog.py:20 | __init__ | Anonymized telemetry enabled. See                     https://docs.trychroma.com/telemetry for more information.\n"
     ]
    }
   ],
   "source": [
    "client = chromadb.PersistentClient(path=\"ChromaDB/chroma.db\")\n",
    "google_ef  = embedding_functions.GoogleGenerativeAiEmbeddingFunction(api_key=\"AIzaSyCP8PWutyKj8N_sM-v1XyPB54_YswnzxYc\")\n",
    "collection = client.get_collection(\"Tech\", embedding_function=google_ef)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scraping Tech Crunch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-24 16:37:59,327 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://techcrunch.com/news-sitemap.xml\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loc</th>\n",
       "      <th>news</th>\n",
       "      <th>news_publication</th>\n",
       "      <th>publication_name</th>\n",
       "      <th>publication_language</th>\n",
       "      <th>news_publication_date</th>\n",
       "      <th>news_title</th>\n",
       "      <th>sitemap</th>\n",
       "      <th>sitemap_size_mb</th>\n",
       "      <th>download_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://techcrunch.com/2024/02/24/google-relea...</td>\n",
       "      <td>\\n\\t\\t</td>\n",
       "      <td>\\n\\t\\t\\t</td>\n",
       "      <td>TechCrunch</td>\n",
       "      <td>en</td>\n",
       "      <td>2024-02-24T21:15:00+00:00</td>\n",
       "      <td>Google releases new open LLMs, Rivian lays off...</td>\n",
       "      <td>https://techcrunch.com/news-sitemap.xml</td>\n",
       "      <td>0.015626</td>\n",
       "      <td>2024-02-24 22:37:59.329332+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://techcrunch.com/2024/02/24/with-liquidi...</td>\n",
       "      <td>\\n\\t\\t</td>\n",
       "      <td>\\n\\t\\t\\t</td>\n",
       "      <td>TechCrunch</td>\n",
       "      <td>en</td>\n",
       "      <td>2024-02-24T18:05:20+00:00</td>\n",
       "      <td>With liquidity rare, VCs may get creative to r...</td>\n",
       "      <td>https://techcrunch.com/news-sitemap.xml</td>\n",
       "      <td>0.015626</td>\n",
       "      <td>2024-02-24 22:37:59.329332+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://techcrunch.com/2024/02/24/equity-shot-...</td>\n",
       "      <td>\\n\\t\\t</td>\n",
       "      <td>\\n\\t\\t\\t</td>\n",
       "      <td>TechCrunch</td>\n",
       "      <td>en</td>\n",
       "      <td>2024-02-24T18:05:00+00:00</td>\n",
       "      <td>Equity Shot: All about the Reddit IPO!</td>\n",
       "      <td>https://techcrunch.com/news-sitemap.xml</td>\n",
       "      <td>0.015626</td>\n",
       "      <td>2024-02-24 22:37:59.329332+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://techcrunch.com/2024/02/24/stellantis-c...</td>\n",
       "      <td>\\n\\t\\t</td>\n",
       "      <td>\\n\\t\\t\\t</td>\n",
       "      <td>TechCrunch</td>\n",
       "      <td>en</td>\n",
       "      <td>2024-02-24T16:00:01+00:00</td>\n",
       "      <td>Stellantis CEO says there&amp;#8217;s still life i...</td>\n",
       "      <td>https://techcrunch.com/news-sitemap.xml</td>\n",
       "      <td>0.015626</td>\n",
       "      <td>2024-02-24 22:37:59.329332+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://techcrunch.com/2024/02/24/this-week-in...</td>\n",
       "      <td>\\n\\t\\t</td>\n",
       "      <td>\\n\\t\\t\\t</td>\n",
       "      <td>TechCrunch</td>\n",
       "      <td>en</td>\n",
       "      <td>2024-02-24T14:45:12+00:00</td>\n",
       "      <td>This Week in AI: Addressing racism in AI image...</td>\n",
       "      <td>https://techcrunch.com/news-sitemap.xml</td>\n",
       "      <td>0.015626</td>\n",
       "      <td>2024-02-24 22:37:59.329332+00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 loc    news news_publication  \\\n",
       "0  https://techcrunch.com/2024/02/24/google-relea...  \\n\\t\\t         \\n\\t\\t\\t   \n",
       "1  https://techcrunch.com/2024/02/24/with-liquidi...  \\n\\t\\t         \\n\\t\\t\\t   \n",
       "2  https://techcrunch.com/2024/02/24/equity-shot-...  \\n\\t\\t         \\n\\t\\t\\t   \n",
       "3  https://techcrunch.com/2024/02/24/stellantis-c...  \\n\\t\\t         \\n\\t\\t\\t   \n",
       "4  https://techcrunch.com/2024/02/24/this-week-in...  \\n\\t\\t         \\n\\t\\t\\t   \n",
       "\n",
       "  publication_name publication_language      news_publication_date  \\\n",
       "0       TechCrunch                   en  2024-02-24T21:15:00+00:00   \n",
       "1       TechCrunch                   en  2024-02-24T18:05:20+00:00   \n",
       "2       TechCrunch                   en  2024-02-24T18:05:00+00:00   \n",
       "3       TechCrunch                   en  2024-02-24T16:00:01+00:00   \n",
       "4       TechCrunch                   en  2024-02-24T14:45:12+00:00   \n",
       "\n",
       "                                          news_title  \\\n",
       "0  Google releases new open LLMs, Rivian lays off...   \n",
       "1  With liquidity rare, VCs may get creative to r...   \n",
       "2             Equity Shot: All about the Reddit IPO!   \n",
       "3  Stellantis CEO says there&#8217;s still life i...   \n",
       "4  This Week in AI: Addressing racism in AI image...   \n",
       "\n",
       "                                   sitemap  sitemap_size_mb  \\\n",
       "0  https://techcrunch.com/news-sitemap.xml         0.015626   \n",
       "1  https://techcrunch.com/news-sitemap.xml         0.015626   \n",
       "2  https://techcrunch.com/news-sitemap.xml         0.015626   \n",
       "3  https://techcrunch.com/news-sitemap.xml         0.015626   \n",
       "4  https://techcrunch.com/news-sitemap.xml         0.015626   \n",
       "\n",
       "                     download_date  \n",
       "0 2024-02-24 22:37:59.329332+00:00  \n",
       "1 2024-02-24 22:37:59.329332+00:00  \n",
       "2 2024-02-24 22:37:59.329332+00:00  \n",
       "3 2024-02-24 22:37:59.329332+00:00  \n",
       "4 2024-02-24 22:37:59.329332+00:00  "
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "techcrunch = adv.sitemap_to_df('https://techcrunch.com/news-sitemap.xml', recursive=True)\n",
    "\n",
    "techcrunch.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Google releases new open LLMs, Rivian lays off staff and Signal rolls out usernames | TechCrunch\n",
      "\n",
      "Welcome, folks, to Week in Review (WiR), TechCrunch’s regular newsletter covering noteworthy happenings in the tech industry.\n",
      "This week, Google launched two new open large language models, Gemma 2B and Gemma 7B, in its continued bid for generative AI dominance. The company, which describes the LLMs as “inspired by Gemini,” its flagship family of GenAI models, made each available for commercial and research usage.\n",
      "Elsewhere, embattled database company MariaDB revealed that it could be taken private in a $37 million deal. The nonbinding proposal comes 14 months after MariaDB went public via SPAC, Paul writes.\n",
      "Lots else happened. We recap it all in this edition of WiR — but first, a reminder to sign up to receive the WiR newsletter in your inbox every Saturday.\n",
      "News\n",
      "Tragedy: Earlier this week, Marco Troper, the 19-year-old son of former YouTube CEO Susan Wojcicki, was found dead at UC Berkeley of an apparent drug overdose.\n",
      "Cake sale: The future of bankrupt electric motorbike startup Cake is still uncertain, but the majority of its U.S. inventory is going to a man in Florida, Sean reports.\n",
      "Change Healthcare hit: Change Healthcare, one of the largest healthcare tech companies in the U.S., confirmed that a cyberattack on its systems occurred recently.\n",
      "Rivan layoffs: Rivian is laying off 10% of its salaried workforce in a bid to cut costs in an increasingly tough market for electric vehicles, putting even more pressure on its future, more affordable EV called the R2.\n",
      "Dunzo buyout: Indian e-commerce giant Flipkart has held discussions in recent weeks about potentially acquiring Dunzo, the hyperlocal delivery startup backed by Reliance Retail, Manish reports.\n",
      "More privacy: Signal now lets you keep your phone number private with the launch of usernames.\n",
      "YouTube triumphant: YouTube dominates TV streaming in the U.S., per Nielsen’s latest report.\n",
      "Valuation cut: Byju’s says its recently launched $200 million rights issue has been fully subscribed, but the startup’s founder urged some of its major investors to participate amid a rift between the edtech group and some of its largest shareholders, Manish writes.\n",
      "Funding\n",
      "Lucrative grooming: France-based Planity has raised $48 million to expand its software-as-a-service product for hair salons.\n",
      "Brick-laying robots: Dutch startup Monumental landed a $25 million tranche for its robotic carts and arms that help lay bricks for construction.\n",
      "Analysis\n",
      "Segment under threat: Alex and Ron take a look at the reasons why Segment, the customer data company, could be put up for sale by corporate parent Twilio. Activist pressure has a lot to do with it.\n",
      "Podcasts\n",
      "On Equity, the crew talked about Reddit’s upcoming IPO, along with some impressive raises for edtech startup Loora, Bioptimus and Dili.\n",
      "Found dove into the enormous web of the wedding industry with Shan-Lyn Ma, the co-founder and CEO of Zola. Ma talked about why she decided to launch the business after trying to buy a gift for a friend and realizing that wedding registries were still living in the past.\n",
      "And Chain Reaction had on Steve Kaczynski, co-author of the book “The Everything Token” and co-host of the web3 morning show “Coffee with Captain.” \n",
      "Bonus round\n",
      "New AI chip venture: SoftBank Group’s Masayoshi Son is reportedly seeking $100 billion to build a new venture that’d compete with the likes of Nvidia in the area of AI chips.\n",
      "OnePlus returns to watches: OnePlus’ upcoming Watch 2 — the company’s first smartwatch in years — promises a jaw-dropping 100 hours “in full Smart Mode.”\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "35"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get content of one url\n",
    "url = techcrunch.loc[0, 'loc']\n",
    "\n",
    "# Get the content of the url\n",
    "response = requests.get(url)\n",
    "html = response.content\n",
    "\n",
    "# Parse the html content\n",
    "soup = BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "# Get the title of the page\n",
    "title = soup.title.text\n",
    "\n",
    "# Get the content of the page\n",
    "content = soup.find('div', class_='article-content').text\n",
    "\n",
    "# Print the title and content\n",
    "print(title)\n",
    "print(content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding it to ChromaDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(techcrunch)):\n",
    "    url = techcrunch.loc[i, 'loc']\n",
    "    response = requests.get(url)\n",
    "    html = response.content\n",
    "    soup = BeautifulSoup(html, 'html.parser')\n",
    "    title = soup.title.text\n",
    "    content = soup.find('div', class_='article-content').text\n",
    "    id += 1\n",
    "    collection.add(\n",
    "        ids=url,\n",
    "        documents=content[:900],\n",
    "        metadatas={\n",
    "            \"title\": title,\n",
    "            \"url\": url,\n",
    "            \"source\": \"TechCrunch\",\n",
    "        }\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scraping the Verge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-24 13:00:25,550 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2011/4\n",
      "2024-02-24 13:00:25,554 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2011/7\n",
      "2024-02-24 13:00:25,558 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2011/6\n",
      "2024-02-24 13:00:25,703 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2011/8\n",
      "2024-02-24 13:00:25,789 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2011/5\n",
      "2024-02-24 13:00:26,738 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2011/9\n",
      "2024-02-24 13:00:28,043 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2011/10\n",
      "2024-02-24 13:00:28,116 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2011/12\n",
      "2024-02-24 13:00:28,330 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2011/11\n",
      "2024-02-24 13:00:28,622 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2012/4\n",
      "2024-02-24 13:00:30,810 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2012/9\n",
      "2024-02-24 13:00:30,896 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2012/3\n",
      "2024-02-24 13:00:31,027 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2012/7\n",
      "2024-02-24 13:00:31,496 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2012/6\n",
      "2024-02-24 13:00:31,566 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2012/8\n",
      "2024-02-24 13:00:32,282 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2012/2\n",
      "2024-02-24 13:00:33,165 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2012/12\n",
      "2024-02-24 13:00:34,693 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2013/3\n",
      "2024-02-24 13:00:35,210 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2012/5\n",
      "2024-02-24 13:00:35,388 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2013/1\n",
      "2024-02-24 13:00:35,408 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2012/10\n",
      "2024-02-24 13:00:35,629 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2013/4\n",
      "2024-02-24 13:00:35,813 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2012/11\n",
      "2024-02-24 13:00:36,398 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2012/1\n",
      "2024-02-24 13:00:37,243 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2013/2\n",
      "2024-02-24 13:00:37,753 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2013/5\n",
      "2024-02-24 13:00:37,935 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2013/6\n",
      "2024-02-24 13:00:37,950 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2013/7\n",
      "2024-02-24 13:00:38,491 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2013/10\n",
      "2024-02-24 13:00:38,863 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2013/11\n",
      "2024-02-24 13:00:39,127 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2013/12\n",
      "2024-02-24 13:00:39,644 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2014/3\n",
      "2024-02-24 13:00:40,210 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2014/2\n",
      "2024-02-24 13:00:40,373 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2014/4\n",
      "2024-02-24 13:00:40,581 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2014/1\n",
      "2024-02-24 13:00:40,697 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2013/8\n",
      "2024-02-24 13:00:40,891 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2014/5\n",
      "2024-02-24 13:00:41,466 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2014/7\n",
      "2024-02-24 13:00:41,676 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2014/6\n",
      "2024-02-24 13:00:41,913 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2013/9\n",
      "2024-02-24 13:00:42,233 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2014/8\n",
      "2024-02-24 13:00:42,279 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2014/11\n",
      "2024-02-24 13:00:42,622 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2014/10\n",
      "2024-02-24 13:00:42,695 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2014/9\n",
      "2024-02-24 13:00:43,918 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2014/12\n",
      "2024-02-24 13:00:43,921 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2015/2\n",
      "2024-02-24 13:00:44,325 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2015/3\n",
      "2024-02-24 13:00:45,356 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2015/6\n",
      "2024-02-24 13:00:46,175 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2015/8\n",
      "2024-02-24 13:00:46,995 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2015/1\n",
      "2024-02-24 13:00:47,002 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2015/5\n",
      "2024-02-24 13:00:47,288 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2015/4\n",
      "2024-02-24 13:00:47,555 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2015/11\n",
      "2024-02-24 13:00:48,531 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2015/7\n",
      "2024-02-24 13:00:50,069 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2016/2\n",
      "2024-02-24 13:00:50,097 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2016/3\n",
      "2024-02-24 13:00:50,336 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2015/9\n",
      "2024-02-24 13:00:50,432 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2015/10\n",
      "2024-02-24 13:00:50,474 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2016/1\n",
      "2024-02-24 13:00:51,592 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2015/12\n",
      "2024-02-24 13:00:51,985 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2016/5\n",
      "2024-02-24 13:00:52,623 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2016/4\n",
      "2024-02-24 13:00:52,741 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2016/10\n",
      "2024-02-24 13:00:52,955 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2016/9\n",
      "2024-02-24 13:00:53,737 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2016/12\n",
      "2024-02-24 13:00:54,052 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2016/6\n",
      "2024-02-24 13:00:55,378 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2017/2\n",
      "2024-02-24 13:00:55,405 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2017/3\n",
      "2024-02-24 13:00:55,490 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2016/11\n",
      "2024-02-24 13:00:55,576 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2017/1\n",
      "2024-02-24 13:00:55,595 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2016/7\n",
      "2024-02-24 13:00:56,091 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2016/8\n",
      "2024-02-24 13:00:56,599 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2017/5\n",
      "2024-02-24 13:00:57,523 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2017/7\n",
      "2024-02-24 13:00:57,905 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2017/8\n",
      "2024-02-24 13:00:57,931 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2017/9\n",
      "2024-02-24 13:00:58,219 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2017/11\n",
      "2024-02-24 13:00:58,490 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2017/12\n",
      "2024-02-24 13:00:58,582 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2017/4\n",
      "2024-02-24 13:01:00,111 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2017/10\n",
      "2024-02-24 13:01:00,296 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2018/3\n",
      "2024-02-24 13:01:00,354 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2018/1\n",
      "2024-02-24 13:01:01,324 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2017/6\n",
      "2024-02-24 13:01:02,246 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2018/7\n",
      "2024-02-24 13:01:02,399 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2018/2\n",
      "2024-02-24 13:01:02,419 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2018/9\n",
      "2024-02-24 13:01:02,450 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2018/8\n",
      "2024-02-24 13:01:02,924 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2018/4\n",
      "2024-02-24 13:01:03,814 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2018/10\n",
      "2024-02-24 13:01:03,983 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2018/12\n",
      "2024-02-24 13:01:04,036 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2018/6\n",
      "2024-02-24 13:01:04,305 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2019/1\n",
      "2024-02-24 13:01:06,139 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2019/5\n",
      "2024-02-24 13:01:06,197 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2018/5\n",
      "2024-02-24 13:01:06,215 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2019/6\n",
      "2024-02-24 13:01:06,380 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2019/7\n",
      "2024-02-24 13:01:07,198 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2019/3\n",
      "2024-02-24 13:01:07,300 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2019/2\n",
      "2024-02-24 13:01:07,484 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2018/11\n",
      "2024-02-24 13:01:08,193 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2019/10\n",
      "2024-02-24 13:01:08,212 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2019/8\n",
      "2024-02-24 13:01:08,212 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2019/11\n",
      "2024-02-24 13:01:08,265 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2019/9\n",
      "2024-02-24 13:01:08,790 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2019/12\n",
      "2024-02-24 13:01:09,909 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2020/1\n",
      "2024-02-24 13:01:09,926 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2020/2\n",
      "2024-02-24 13:01:11,144 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2020/6\n",
      "2024-02-24 13:01:11,534 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2019/4\n",
      "2024-02-24 13:01:11,852 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2020/4\n",
      "2024-02-24 13:01:11,892 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2020/8\n",
      "2024-02-24 13:01:12,580 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2020/5\n",
      "2024-02-24 13:01:12,724 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2020/7\n",
      "2024-02-24 13:01:12,945 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2021/3\n",
      "2024-02-24 13:01:13,063 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2021/4\n",
      "2024-02-24 13:01:13,185 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2020/11\n",
      "2024-02-24 13:01:13,186 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2021/5\n",
      "2024-02-24 13:01:13,288 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2021/6\n",
      "2024-02-24 13:01:13,308 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2021/7\n",
      "2024-02-24 13:01:13,385 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2020/10\n",
      "2024-02-24 13:01:13,468 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2021/9\n",
      "2024-02-24 13:01:13,497 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2021/8\n",
      "2024-02-24 13:01:13,513 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2021/10\n",
      "2024-02-24 13:01:13,587 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2021/11\n",
      "2024-02-24 13:01:13,604 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2022/1\n",
      "2024-02-24 13:01:13,612 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2021/12\n",
      "2024-02-24 13:01:13,660 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2020/3\n",
      "2024-02-24 13:01:13,702 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2022/4\n",
      "2024-02-24 13:01:13,713 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2022/3\n",
      "2024-02-24 13:01:13,758 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2022/5\n",
      "2024-02-24 13:01:13,921 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2022/6\n",
      "2024-02-24 13:01:13,932 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2022/7\n",
      "2024-02-24 13:01:13,936 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2020/12\n",
      "2024-02-24 13:01:14,015 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2022/10\n",
      "2024-02-24 13:01:14,039 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2022/9\n",
      "2024-02-24 13:01:14,073 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2022/11\n",
      "2024-02-24 13:01:14,131 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2020/9\n",
      "2024-02-24 13:01:14,139 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2022/2\n",
      "2024-02-24 13:01:14,159 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2023/1\n",
      "2024-02-24 13:01:14,166 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2022/12\n",
      "2024-02-24 13:01:14,255 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2023/4\n",
      "2024-02-24 13:01:14,272 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2023/6\n",
      "2024-02-24 13:01:14,283 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2023/5\n",
      "2024-02-24 13:01:14,433 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2023/7\n",
      "2024-02-24 13:01:14,496 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2023/8\n",
      "2024-02-24 13:01:14,534 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2023/10\n",
      "2024-02-24 13:01:14,560 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2023/2\n",
      "2024-02-24 13:01:14,589 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2023/11\n",
      "2024-02-24 13:01:14,643 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2024/1\n",
      "2024-02-24 13:01:14,677 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2024/2\n",
      "2024-02-24 13:01:14,691 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2022/8\n",
      "2024-02-24 13:01:14,753 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2023/12\n",
      "2024-02-24 13:01:14,771 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/groups\n",
      "2024-02-24 13:01:14,783 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/authors\n",
      "2024-02-24 13:01:14,811 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/video/1\n",
      "2024-02-24 13:01:14,847 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/video/3\n",
      "2024-02-24 13:01:14,862 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/video/2\n",
      "2024-02-24 13:01:15,053 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/google_news\n",
      "2024-02-24 13:01:15,073 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/video/6\n",
      "2024-02-24 13:01:15,101 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2023/3\n",
      "2024-02-24 13:01:15,112 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/video/5\n",
      "2024-02-24 13:01:15,726 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/entries/2023/9\n",
      "2024-02-24 13:01:16,036 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2021/1\n",
      "2024-02-24 13:01:16,090 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 429: Too Many Requests https://www.theverge.com/sitemaps/video/4\n",
      "2024-02-24 13:01:16,651 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.theverge.com/sitemaps/entries/2021/2\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loc</th>\n",
       "      <th>lastmod</th>\n",
       "      <th>sitemap</th>\n",
       "      <th>etag</th>\n",
       "      <th>sitemap_size_mb</th>\n",
       "      <th>download_date</th>\n",
       "      <th>errors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.theverge.com/2011/04/30/carriers-a...</td>\n",
       "      <td>2011-04-30 22:13:29+00:00</td>\n",
       "      <td>https://www.theverge.com/sitemaps/entries/2011/4</td>\n",
       "      <td>W/\"e9808ba57c91cd4a51a4e79f9d2fadbb\"</td>\n",
       "      <td>0.00783</td>\n",
       "      <td>2024-02-24 19:00:25.562534+00:00</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.theverge.com/2011/04/29/podcast-00...</td>\n",
       "      <td>2011-04-29 21:21:11+00:00</td>\n",
       "      <td>https://www.theverge.com/sitemaps/entries/2011/4</td>\n",
       "      <td>W/\"e9808ba57c91cd4a51a4e79f9d2fadbb\"</td>\n",
       "      <td>0.00783</td>\n",
       "      <td>2024-02-24 19:00:25.562534+00:00</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.theverge.com/2011/10/23/2509628/tw...</td>\n",
       "      <td>2011-04-29 21:15:00+00:00</td>\n",
       "      <td>https://www.theverge.com/sitemaps/entries/2011/4</td>\n",
       "      <td>W/\"e9808ba57c91cd4a51a4e79f9d2fadbb\"</td>\n",
       "      <td>0.00783</td>\n",
       "      <td>2024-02-24 19:00:25.562534+00:00</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.theverge.com/2011/04/29/samsung-su...</td>\n",
       "      <td>2011-04-29 20:20:16+00:00</td>\n",
       "      <td>https://www.theverge.com/sitemaps/entries/2011/4</td>\n",
       "      <td>W/\"e9808ba57c91cd4a51a4e79f9d2fadbb\"</td>\n",
       "      <td>0.00783</td>\n",
       "      <td>2024-02-24 19:00:25.562534+00:00</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.theverge.com/2011/10/23/2509289/hu...</td>\n",
       "      <td>2011-04-29 17:43:00+00:00</td>\n",
       "      <td>https://www.theverge.com/sitemaps/entries/2011/4</td>\n",
       "      <td>W/\"e9808ba57c91cd4a51a4e79f9d2fadbb\"</td>\n",
       "      <td>0.00783</td>\n",
       "      <td>2024-02-24 19:00:25.562534+00:00</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 loc  \\\n",
       "0  https://www.theverge.com/2011/04/30/carriers-a...   \n",
       "1  https://www.theverge.com/2011/04/29/podcast-00...   \n",
       "2  https://www.theverge.com/2011/10/23/2509628/tw...   \n",
       "3  https://www.theverge.com/2011/04/29/samsung-su...   \n",
       "4  https://www.theverge.com/2011/10/23/2509289/hu...   \n",
       "\n",
       "                    lastmod                                           sitemap  \\\n",
       "0 2011-04-30 22:13:29+00:00  https://www.theverge.com/sitemaps/entries/2011/4   \n",
       "1 2011-04-29 21:21:11+00:00  https://www.theverge.com/sitemaps/entries/2011/4   \n",
       "2 2011-04-29 21:15:00+00:00  https://www.theverge.com/sitemaps/entries/2011/4   \n",
       "3 2011-04-29 20:20:16+00:00  https://www.theverge.com/sitemaps/entries/2011/4   \n",
       "4 2011-04-29 17:43:00+00:00  https://www.theverge.com/sitemaps/entries/2011/4   \n",
       "\n",
       "                                   etag  sitemap_size_mb  \\\n",
       "0  W/\"e9808ba57c91cd4a51a4e79f9d2fadbb\"          0.00783   \n",
       "1  W/\"e9808ba57c91cd4a51a4e79f9d2fadbb\"          0.00783   \n",
       "2  W/\"e9808ba57c91cd4a51a4e79f9d2fadbb\"          0.00783   \n",
       "3  W/\"e9808ba57c91cd4a51a4e79f9d2fadbb\"          0.00783   \n",
       "4  W/\"e9808ba57c91cd4a51a4e79f9d2fadbb\"          0.00783   \n",
       "\n",
       "                     download_date errors  \n",
       "0 2024-02-24 19:00:25.562534+00:00    NaN  \n",
       "1 2024-02-24 19:00:25.562534+00:00    NaN  \n",
       "2 2024-02-24 19:00:25.562534+00:00    NaN  \n",
       "3 2024-02-24 19:00:25.562534+00:00    NaN  \n",
       "4 2024-02-24 19:00:25.562534+00:00    NaN  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "verge = adv.sitemap_to_df('https://www.theverge.com/sitemaps', recursive=True)\n",
    "\n",
    "verge.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.theverge.com/2011/04/30/carriers-act-swiftly-save-low-end-smartphone\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('Carriers: act swiftly if you want to save the low-end smartphone - The Verge',\n",
       " 'For years, I\\'ve been talking about the effects of price pressure on the subsidized smartphone business -- the phenomenon that, when left unchecked, results in too many market segments squeezed into far too narrow of a price range. In turn, that leaves low-end devices priced right out of the market altogether. Of course, smartphones pervade every nook and cranny of your carrier\\'s lineup more completely nowadays than ever before, and I\\'ve found that it\\'s leading to some truly strange pricing problems.\\n\\nThat was really driven home for me today when I took a look at the first page of of Android devices on T-Mobile\\'s online store. In the span of just $50, you range all the way from the lowly Comet -- sourced from Huawei with a QVGA display -- up to the relatively powerful, capable, and competent myTouch 4G with HSPA+ and a front-facing cam. Granted, the myTouch is refurbished, but you\\'ve still got access to the Optimus T and myTouch 3G for free -- and both of those devices are head and shoulders above the Comet, Charm, CLIQ, or CLIQ XT. Every one of those currently go for $0 on contract. And when you look at the total cost of ownership over the span of a two-year commitment, the difference in cost between a free device and a $50 one is all but meaningless.\\n\\nIt\\'s like your realtor showing you two homes: a one-bedroom shack with a leaky roof for $100,000, or an eight-bedroom mansion with a pool for $110,000, both with the same maintenance costs over time (far-fetched, I know, but stay with me on the analogy). Which would you choose? Even if you had to stretch a little bit, you\\'d obviously aim for the mansion. And what\\'s especially troubling is that for your average consumer, the difference in quality and capability between a Comet and an Optimus T is far less obvious than the difference between a shack and a mansion at the point of purchase. In fact, I\\'d argue that the obscurity is the one and only reason devices like the Comet have a market right now.\\n\\nSo far, American carriers seem reluctant to deal with it, but here\\'s the thing: the answer doesn\\'t simply have to be for low-end smartphones to go away. Not everyone needs an iPhone, an Atrix, or a G2x, but it is the responsibility of the carriers to reward their customers appropriately for choosing cheaper devices upfront. That\\'s the place where they\\'re failing. I think there are a couple possible options on the table:\\n\\n*  Low-end smartphones could be offered with cheaper, exclusive plans. T-Mobile tried this with the Nokia Nuron, which could be activated with a relatively cheap $10 unlimited data plan typically reserved for dumbphones. Over the course of your contract, the benefits really start to add up -- and I bet that carriers\\' internal stats support my belief that owners of devices like the Comet use significantly less data than owners of midrange and high-end devices (those in the $100 to $300 range). The downside is that there would be a constant barrage of controversial gray areas about what phones should be eligible for the cheaper plans... and plus, as devices get cheaper over time, you\\'d need to decide whether to make them eligible midstream. Pretty hairy.\\n*  More creative subsidies. Right now, there seems to be an implicit understanding that the floor is $0 on contract. But why? If the Optimus T is free, I think you can make the argument that the Comet is worth about $-50, so either pay the buyer upfront (say, with a prepaid card) or -- probably the more compelling and manageable option -- offer a bill credit. So, for example, if you buy the Comet, you pay nothing upfront and the first $50 worth of service would be covered. Sprint ran a promotion recently that involved bill credits, so I know for a fact these carriers have the technical capability to manage an option like this.\\n\\nThe bottom line is that the price pressure phenomenon is entirely artificial and ends up short-changing bargain shoppers who deal with an enormous range of quality in the span of a couple $20 bills when they walk into the store. As with any product, you should be able to figure out very quickly how capable a product is by its relative pricing -- and in an age where even $250 is \"pushing it\" for a high-end model on contract, these guys are going to need to get creative.\\n')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get content of one url\n",
    "url = verge.loc[0, 'loc']\n",
    "\n",
    "# Get the content of the url\n",
    "response = requests.get(url)\n",
    "print(url)\n",
    "html = response.content\n",
    "\n",
    "# Parse the html content\n",
    "soup = BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "# Get the title of the page\n",
    "title = soup.title.text\n",
    "\n",
    "# Get the content of the page\n",
    "content = soup.find('script', type='application/ld+json').string\n",
    "\n",
    "data = json.loads(content)\n",
    "\n",
    "article_body = data.get(\"articleBody\", \"Text not found\")\n",
    "\n",
    "title, article_body\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding Verge to ChromaDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-24 16:49:32,542 | WARNING | sqlite.py:284 | _insert_record | Insert of existing embedding ID: https://www.theverge.com/2011/04/30/carriers-act-swiftly-save-low-end-smartphone\n",
      "2024-02-24 16:49:32,543 | WARNING | local_persistent_hnsw.py:271 | _write_records | Add of existing embedding ID: https://www.theverge.com/2011/04/30/carriers-act-swiftly-save-low-end-smartphone\n",
      "2024-02-24 16:50:20,367 | WARNING | sqlite.py:284 | _insert_record | Insert of existing embedding ID: https://www.theverge.com/2011/04/29/samsung-sues-apple-infringing-10-patents-closer\n",
      "2024-02-24 16:50:20,371 | WARNING | local_persistent_hnsw.py:271 | _write_records | Add of existing embedding ID: https://www.theverge.com/2011/04/29/samsung-sues-apple-infringing-10-patents-closer\n",
      "2024-02-24 16:51:10,785 | WARNING | sqlite.py:284 | _insert_record | Insert of existing embedding ID: https://www.theverge.com/2011/4/28/2514900/sony-its-time-to-act-like-sony-again\n",
      "2024-02-24 16:51:10,786 | WARNING | local_persistent_hnsw.py:271 | _write_records | Add of existing embedding ID: https://www.theverge.com/2011/4/28/2514900/sony-its-time-to-act-like-sony-again\n",
      "2024-02-24 16:51:11,047 | WARNING | sqlite.py:284 | _insert_record | Insert of existing embedding ID: https://www.theverge.com/2011/04/28/verizon-endures-full-day-lte\n",
      "2024-02-24 16:51:11,048 | WARNING | local_persistent_hnsw.py:271 | _write_records | Add of existing embedding ID: https://www.theverge.com/2011/04/28/verizon-endures-full-day-lte\n",
      "2024-02-24 16:51:11,314 | WARNING | sqlite.py:284 | _insert_record | Insert of existing embedding ID: https://www.theverge.com/2011/10/23/2508942/acer-iconia-tab-a500-review\n",
      "2024-02-24 16:51:11,315 | WARNING | local_persistent_hnsw.py:271 | _write_records | Add of existing embedding ID: https://www.theverge.com/2011/10/23/2508942/acer-iconia-tab-a500-review\n",
      "2024-02-24 16:51:11,584 | WARNING | sqlite.py:284 | _insert_record | Insert of existing embedding ID: https://www.theverge.com/2011/04/27/google-docs-for-android\n",
      "2024-02-24 16:51:11,584 | WARNING | local_persistent_hnsw.py:271 | _write_records | Add of existing embedding ID: https://www.theverge.com/2011/04/27/google-docs-for-android\n",
      "2024-02-24 16:51:11,858 | WARNING | sqlite.py:284 | _insert_record | Insert of existing embedding ID: https://www.theverge.com/2011/04/27/nokia-anxious-shuffle-symbian-coil-outsourcing-development-accenture\n",
      "2024-02-24 16:51:11,859 | WARNING | local_persistent_hnsw.py:271 | _write_records | Add of existing embedding ID: https://www.theverge.com/2011/04/27/nokia-anxious-shuffle-symbian-coil-outsourcing-development-accenture\n",
      "2024-02-24 16:51:12,138 | WARNING | sqlite.py:284 | _insert_record | Insert of existing embedding ID: https://www.theverge.com/2011/04/27/apple-posts-iphone-location-tracking-qa-complicated-ios-update-coming\n",
      "2024-02-24 16:51:12,139 | WARNING | local_persistent_hnsw.py:271 | _write_records | Add of existing embedding ID: https://www.theverge.com/2011/04/27/apple-posts-iphone-location-tracking-qa-complicated-ios-update-coming\n",
      "2024-02-24 16:51:12,404 | WARNING | sqlite.py:284 | _insert_record | Insert of existing embedding ID: https://www.theverge.com/2011/04/27/hulu-plus-on-playstation-network-xbox-360\n",
      "2024-02-24 16:51:12,405 | WARNING | local_persistent_hnsw.py:271 | _write_records | Add of existing embedding ID: https://www.theverge.com/2011/04/27/hulu-plus-on-playstation-network-xbox-360\n",
      "2024-02-24 16:51:12,679 | WARNING | sqlite.py:284 | _insert_record | Insert of existing embedding ID: https://www.theverge.com/2011/04/26/playstation-network-damage-report\n",
      "2024-02-24 16:51:12,680 | WARNING | local_persistent_hnsw.py:271 | _write_records | Add of existing embedding ID: https://www.theverge.com/2011/04/26/playstation-network-damage-report\n"
     ]
    },
    {
     "ename": "DeadlineExceeded",
     "evalue": "504 Deadline Exceeded",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31m_InactiveRpcError\u001b[0m                         Traceback (most recent call last)",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/google/api_core/grpc_helpers.py:79\u001b[0m, in \u001b[0;36m_wrap_unary_errors.<locals>.error_remapped_callable\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     78\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 79\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcallable_\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     80\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m grpc\u001b[38;5;241m.\u001b[39mRpcError \u001b[38;5;28;01mas\u001b[39;00m exc:\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/grpc/_channel.py:1160\u001b[0m, in \u001b[0;36m_UnaryUnaryMultiCallable.__call__\u001b[0;34m(self, request, timeout, metadata, credentials, wait_for_ready, compression)\u001b[0m\n\u001b[1;32m   1154\u001b[0m (\n\u001b[1;32m   1155\u001b[0m     state,\n\u001b[1;32m   1156\u001b[0m     call,\n\u001b[1;32m   1157\u001b[0m ) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_blocking(\n\u001b[1;32m   1158\u001b[0m     request, timeout, metadata, credentials, wait_for_ready, compression\n\u001b[1;32m   1159\u001b[0m )\n\u001b[0;32m-> 1160\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_end_unary_response_blocking\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcall\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/grpc/_channel.py:1003\u001b[0m, in \u001b[0;36m_end_unary_response_blocking\u001b[0;34m(state, call, with_call, deadline)\u001b[0m\n\u001b[1;32m   1002\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1003\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m _InactiveRpcError(state)\n",
      "\u001b[0;31m_InactiveRpcError\u001b[0m: <_InactiveRpcError of RPC that terminated with:\n\tstatus = StatusCode.DEADLINE_EXCEEDED\n\tdetails = \"Deadline Exceeded\"\n\tdebug_error_string = \"UNKNOWN:Error received from peer  {grpc_message:\"Deadline Exceeded\", grpc_status:4, created_time:\"2024-02-24T18:40:07.216749-06:00\"}\"\n>",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mDeadlineExceeded\u001b[0m                          Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[131], line 15\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[1;32m     14\u001b[0m \u001b[38;5;28mid\u001b[39m \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[0;32m---> 15\u001b[0m \u001b[43mcollection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43madd\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     16\u001b[0m \u001b[43m    \u001b[49m\u001b[43mids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     17\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdocuments\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43marticle_body\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[38;5;241;43m900\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     18\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmetadatas\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\n\u001b[1;32m     19\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtitle\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mtitle\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     20\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43murl\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     21\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msource\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mTheVerge\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     22\u001b[0m \u001b[43m    \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m     23\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/chromadb/api/models/Collection.py:154\u001b[0m, in \u001b[0;36mCollection.add\u001b[0;34m(self, ids, embeddings, metadatas, documents, images, uris)\u001b[0m\n\u001b[1;32m    151\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m embeddings \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    152\u001b[0m     \u001b[38;5;66;03m# At this point, we know that one of documents or images are provided from the validation above\u001b[39;00m\n\u001b[1;32m    153\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m documents \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 154\u001b[0m         embeddings \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_embed\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdocuments\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    155\u001b[0m     \u001b[38;5;28;01melif\u001b[39;00m images \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    156\u001b[0m         embeddings \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_embed(\u001b[38;5;28minput\u001b[39m\u001b[38;5;241m=\u001b[39mimages)\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/chromadb/api/models/Collection.py:633\u001b[0m, in \u001b[0;36mCollection._embed\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    628\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_embedding_function \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    629\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    630\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYou must provide an embedding function to compute embeddings.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    631\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttps://docs.trychroma.com/embeddings\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    632\u001b[0m     )\n\u001b[0;32m--> 633\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_embedding_function\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/chromadb/api/types.py:193\u001b[0m, in \u001b[0;36mEmbeddingFunction.__init_subclass__.<locals>.__call__\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    192\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m: EmbeddingFunction[D], \u001b[38;5;28minput\u001b[39m: D) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Embeddings:\n\u001b[0;32m--> 193\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mcall\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m    194\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m validate_embeddings(maybe_cast_one_to_many_embedding(result))\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/chromadb/utils/embedding_functions.py:632\u001b[0m, in \u001b[0;36mGoogleGenerativeAiEmbeddingFunction.__call__\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    631\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Documents) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Embeddings:\n\u001b[0;32m--> 632\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m [\n\u001b[1;32m    633\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_genai\u001b[38;5;241m.\u001b[39membed_content(\n\u001b[1;32m    634\u001b[0m             model\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_model_name,\n\u001b[1;32m    635\u001b[0m             content\u001b[38;5;241m=\u001b[39mtext,\n\u001b[1;32m    636\u001b[0m             task_type\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_task_type,\n\u001b[1;32m    637\u001b[0m             title\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_task_title,\n\u001b[1;32m    638\u001b[0m         )[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124membedding\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m    639\u001b[0m         \u001b[38;5;28;01mfor\u001b[39;00m text \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28minput\u001b[39m\n\u001b[1;32m    640\u001b[0m     ]\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/chromadb/utils/embedding_functions.py:633\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    631\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Documents) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Embeddings:\n\u001b[1;32m    632\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m [\n\u001b[0;32m--> 633\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_genai\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43membed_content\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    634\u001b[0m \u001b[43m            \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_model_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    635\u001b[0m \u001b[43m            \u001b[49m\u001b[43mcontent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtext\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    636\u001b[0m \u001b[43m            \u001b[49m\u001b[43mtask_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_task_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    637\u001b[0m \u001b[43m            \u001b[49m\u001b[43mtitle\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_task_title\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    638\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124membedding\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m    639\u001b[0m         \u001b[38;5;28;01mfor\u001b[39;00m text \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28minput\u001b[39m\n\u001b[1;32m    640\u001b[0m     ]\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/google/generativeai/embedding.py:164\u001b[0m, in \u001b[0;36membed_content\u001b[0;34m(model, content, task_type, title, client)\u001b[0m\n\u001b[1;32m    160\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    161\u001b[0m     embedding_request \u001b[38;5;241m=\u001b[39m glm\u001b[38;5;241m.\u001b[39mEmbedContentRequest(\n\u001b[1;32m    162\u001b[0m         model\u001b[38;5;241m=\u001b[39mmodel, content\u001b[38;5;241m=\u001b[39mcontent_types\u001b[38;5;241m.\u001b[39mto_content(content), task_type\u001b[38;5;241m=\u001b[39mtask_type, title\u001b[38;5;241m=\u001b[39mtitle\n\u001b[1;32m    163\u001b[0m     )\n\u001b[0;32m--> 164\u001b[0m     embedding_response \u001b[38;5;241m=\u001b[39m \u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43membed_content\u001b[49m\u001b[43m(\u001b[49m\u001b[43membedding_request\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    165\u001b[0m     embedding_dict \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mtype\u001b[39m(embedding_response)\u001b[38;5;241m.\u001b[39mto_dict(embedding_response)\n\u001b[1;32m    166\u001b[0m     embedding_dict[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124membedding\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m embedding_dict[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124membedding\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalues\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/google/ai/generativelanguage_v1beta/services/generative_service/client.py:968\u001b[0m, in \u001b[0;36mGenerativeServiceClient.embed_content\u001b[0;34m(self, request, model, content, retry, timeout, metadata)\u001b[0m\n\u001b[1;32m    963\u001b[0m metadata \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mtuple\u001b[39m(metadata) \u001b[38;5;241m+\u001b[39m (\n\u001b[1;32m    964\u001b[0m     gapic_v1\u001b[38;5;241m.\u001b[39mrouting_header\u001b[38;5;241m.\u001b[39mto_grpc_metadata(((\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel\u001b[39m\u001b[38;5;124m\"\u001b[39m, request\u001b[38;5;241m.\u001b[39mmodel),)),\n\u001b[1;32m    965\u001b[0m )\n\u001b[1;32m    967\u001b[0m \u001b[38;5;66;03m# Send the request.\u001b[39;00m\n\u001b[0;32m--> 968\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mrpc\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    969\u001b[0m \u001b[43m    \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    970\u001b[0m \u001b[43m    \u001b[49m\u001b[43mretry\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mretry\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    971\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    972\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmetadata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmetadata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    973\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    975\u001b[0m \u001b[38;5;66;03m# Done; return the response.\u001b[39;00m\n\u001b[1;32m    976\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/google/api_core/gapic_v1/method.py:131\u001b[0m, in \u001b[0;36m_GapicCallable.__call__\u001b[0;34m(self, timeout, retry, compression, *args, **kwargs)\u001b[0m\n\u001b[1;32m    128\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compression \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    129\u001b[0m     kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcompression\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m compression\n\u001b[0;32m--> 131\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mwrapped_func\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/google/api_core/retry.py:372\u001b[0m, in \u001b[0;36mRetry.__call__.<locals>.retry_wrapped_func\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    368\u001b[0m target \u001b[38;5;241m=\u001b[39m functools\u001b[38;5;241m.\u001b[39mpartial(func, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    369\u001b[0m sleep_generator \u001b[38;5;241m=\u001b[39m exponential_sleep_generator(\n\u001b[1;32m    370\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_initial, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_maximum, multiplier\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_multiplier\n\u001b[1;32m    371\u001b[0m )\n\u001b[0;32m--> 372\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mretry_target\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    373\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtarget\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    374\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_predicate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    375\u001b[0m \u001b[43m    \u001b[49m\u001b[43msleep_generator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    376\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    377\u001b[0m \u001b[43m    \u001b[49m\u001b[43mon_error\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mon_error\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    378\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/google/api_core/retry.py:207\u001b[0m, in \u001b[0;36mretry_target\u001b[0;34m(target, predicate, sleep_generator, timeout, on_error, **kwargs)\u001b[0m\n\u001b[1;32m    205\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m sleep \u001b[38;5;129;01min\u001b[39;00m sleep_generator:\n\u001b[1;32m    206\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 207\u001b[0m         result \u001b[38;5;241m=\u001b[39m \u001b[43mtarget\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    208\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m inspect\u001b[38;5;241m.\u001b[39misawaitable(result):\n\u001b[1;32m    209\u001b[0m             warnings\u001b[38;5;241m.\u001b[39mwarn(_ASYNC_RETRY_WARNING)\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/google/api_core/timeout.py:120\u001b[0m, in \u001b[0;36mTimeToDeadlineTimeout.__call__.<locals>.func_with_timeout\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    117\u001b[0m     \u001b[38;5;66;03m# Avoid setting negative timeout\u001b[39;00m\n\u001b[1;32m    118\u001b[0m     kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimeout\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmax\u001b[39m(\u001b[38;5;241m0\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout \u001b[38;5;241m-\u001b[39m time_since_first_attempt)\n\u001b[0;32m--> 120\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/google/api_core/grpc_helpers.py:81\u001b[0m, in \u001b[0;36m_wrap_unary_errors.<locals>.error_remapped_callable\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     79\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m callable_(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m     80\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m grpc\u001b[38;5;241m.\u001b[39mRpcError \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[0;32m---> 81\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m exceptions\u001b[38;5;241m.\u001b[39mfrom_grpc_error(exc) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mexc\u001b[39;00m\n",
      "\u001b[0;31mDeadlineExceeded\u001b[0m: 504 Deadline Exceeded"
     ]
    }
   ],
   "source": [
    "for i in range(len(verge)):\n",
    "    url = verge.loc[i, 'loc']\n",
    "    response = requests.get(url)\n",
    "    html = response.content\n",
    "    soup = BeautifulSoup(html, 'html.parser')\n",
    "    title = soup.title.text\n",
    "    content = soup.find('script', type='application/ld+json')\n",
    "    if content is None:\n",
    "        continue\n",
    "    data = json.loads(content.string)\n",
    "    article_body = data.get(\"articleBody\", \"Text not found\")\n",
    "    if content is None or article_body is None or len(article_body) < 900:\n",
    "        continue\n",
    "    id += 1\n",
    "    collection.add(\n",
    "        ids=url,\n",
    "        documents=article_body[:900],\n",
    "        metadatas={\n",
    "            \"title\": title,\n",
    "            \"url\": url,\n",
    "            \"source\": \"TheVerge\",\n",
    "        }\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GeekWire Scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-24 13:16:56,303 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/video-sitemap-1.xml\n",
      "2024-02-24 13:16:57,089 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-3.xml\n",
      "2024-02-24 13:16:57,166 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-6.xml\n",
      "2024-02-24 13:16:57,204 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-4.xml\n",
      "2024-02-24 13:16:57,396 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-8.xml\n",
      "2024-02-24 13:16:57,414 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-12.xml\n",
      "2024-02-24 13:16:57,445 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-5.xml\n",
      "2024-02-24 13:16:57,463 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-9.xml\n",
      "2024-02-24 13:16:57,498 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-2.xml\n",
      "2024-02-24 13:16:57,602 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-13.xml\n",
      "2024-02-24 13:16:57,845 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-16.xml\n",
      "2024-02-24 13:16:58,508 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-7.xml\n",
      "2024-02-24 13:16:58,538 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-1.xml\n",
      "2024-02-24 13:16:58,553 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-17.xml\n",
      "2024-02-24 13:16:58,557 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-11.xml\n",
      "2024-02-24 13:16:58,562 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-14.xml\n",
      "2024-02-24 13:16:58,660 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-15.xml\n",
      "2024-02-24 13:16:58,682 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-18.xml\n",
      "2024-02-24 13:16:58,864 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-10.xml\n",
      "2024-02-24 13:16:59,179 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-19.xml\n",
      "2024-02-24 13:16:59,388 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-23.xml\n",
      "2024-02-24 13:16:59,405 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-21.xml\n",
      "2024-02-24 13:16:59,455 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-20.xml\n",
      "2024-02-24 13:16:59,465 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-24.xml\n",
      "2024-02-24 13:16:59,504 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-25.xml\n",
      "2024-02-24 13:16:59,523 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-22.xml\n",
      "2024-02-24 13:16:59,543 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.geekwire.com/sitemap-26.xml\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loc</th>\n",
       "      <th>lastmod</th>\n",
       "      <th>video_title</th>\n",
       "      <th>video_thumbnail_loc</th>\n",
       "      <th>video_content_loc</th>\n",
       "      <th>sitemap</th>\n",
       "      <th>sitemap_last_modified</th>\n",
       "      <th>sitemap_size_mb</th>\n",
       "      <th>download_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.geekwire.com/2012/geek-inspiring-c...</td>\n",
       "      <td>2012-06-04 00:50:44+00:00</td>\n",
       "      <td>kinect</td>\n",
       "      <td>https://s0.wp.com/i/blank.jpg</td>\n",
       "      <td>https://cdn.geekwire.com/wp-content/uploads/20...</td>\n",
       "      <td>https://www.geekwire.com/video-sitemap-1.xml</td>\n",
       "      <td>2024-02-24 19:16:56</td>\n",
       "      <td>0.021546</td>\n",
       "      <td>2024-02-24 19:16:56.309302+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.geekwire.com/2015/microsoft-at-40-...</td>\n",
       "      <td>2015-04-04 17:09:41+00:00</td>\n",
       "      <td>gat0-vis-019b</td>\n",
       "      <td>https://s0.wp.com/i/blank.jpg</td>\n",
       "      <td>https://cdn.geekwire.com/wp-content/uploads/20...</td>\n",
       "      <td>https://www.geekwire.com/video-sitemap-1.xml</td>\n",
       "      <td>2024-02-24 19:16:56</td>\n",
       "      <td>0.021546</td>\n",
       "      <td>2024-02-24 19:16:56.309302+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.geekwire.com/2015/microsoft-ditche...</td>\n",
       "      <td>2015-08-12 15:59:45+00:00</td>\n",
       "      <td>WindowsIoTCoreSnippetAirHockey_mid</td>\n",
       "      <td>https://s0.wp.com/i/blank.jpg</td>\n",
       "      <td>https://cdn.geekwire.com/wp-content/uploads/20...</td>\n",
       "      <td>https://www.geekwire.com/video-sitemap-1.xml</td>\n",
       "      <td>2024-02-24 19:16:56</td>\n",
       "      <td>0.021546</td>\n",
       "      <td>2024-02-24 19:16:56.309302+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.geekwire.com/2016/pokemon-go-seatt...</td>\n",
       "      <td>2016-07-14 18:20:27+00:00</td>\n",
       "      <td>IMG_1137.MOV</td>\n",
       "      <td>https://s0.wp.com/i/blank.jpg</td>\n",
       "      <td>https://cdn.geekwire.com/wp-content/uploads/20...</td>\n",
       "      <td>https://www.geekwire.com/video-sitemap-1.xml</td>\n",
       "      <td>2024-02-24 19:16:56</td>\n",
       "      <td>0.021546</td>\n",
       "      <td>2024-02-24 19:16:56.309302+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.geekwire.com/2016/microsoft-tries-...</td>\n",
       "      <td>2016-07-27 15:24:15+00:00</td>\n",
       "      <td>Microsoft Pix live image</td>\n",
       "      <td>https://s0.wp.com/i/blank.jpg</td>\n",
       "      <td>https://cdn.geekwire.com/wp-content/uploads/20...</td>\n",
       "      <td>https://www.geekwire.com/video-sitemap-1.xml</td>\n",
       "      <td>2024-02-24 19:16:56</td>\n",
       "      <td>0.021546</td>\n",
       "      <td>2024-02-24 19:16:56.309302+00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 loc  \\\n",
       "0  https://www.geekwire.com/2012/geek-inspiring-c...   \n",
       "1  https://www.geekwire.com/2015/microsoft-at-40-...   \n",
       "2  https://www.geekwire.com/2015/microsoft-ditche...   \n",
       "3  https://www.geekwire.com/2016/pokemon-go-seatt...   \n",
       "4  https://www.geekwire.com/2016/microsoft-tries-...   \n",
       "\n",
       "                    lastmod                         video_title  \\\n",
       "0 2012-06-04 00:50:44+00:00                              kinect   \n",
       "1 2015-04-04 17:09:41+00:00                       gat0-vis-019b   \n",
       "2 2015-08-12 15:59:45+00:00  WindowsIoTCoreSnippetAirHockey_mid   \n",
       "3 2016-07-14 18:20:27+00:00                        IMG_1137.MOV   \n",
       "4 2016-07-27 15:24:15+00:00            Microsoft Pix live image   \n",
       "\n",
       "             video_thumbnail_loc  \\\n",
       "0  https://s0.wp.com/i/blank.jpg   \n",
       "1  https://s0.wp.com/i/blank.jpg   \n",
       "2  https://s0.wp.com/i/blank.jpg   \n",
       "3  https://s0.wp.com/i/blank.jpg   \n",
       "4  https://s0.wp.com/i/blank.jpg   \n",
       "\n",
       "                                   video_content_loc  \\\n",
       "0  https://cdn.geekwire.com/wp-content/uploads/20...   \n",
       "1  https://cdn.geekwire.com/wp-content/uploads/20...   \n",
       "2  https://cdn.geekwire.com/wp-content/uploads/20...   \n",
       "3  https://cdn.geekwire.com/wp-content/uploads/20...   \n",
       "4  https://cdn.geekwire.com/wp-content/uploads/20...   \n",
       "\n",
       "                                        sitemap sitemap_last_modified  \\\n",
       "0  https://www.geekwire.com/video-sitemap-1.xml   2024-02-24 19:16:56   \n",
       "1  https://www.geekwire.com/video-sitemap-1.xml   2024-02-24 19:16:56   \n",
       "2  https://www.geekwire.com/video-sitemap-1.xml   2024-02-24 19:16:56   \n",
       "3  https://www.geekwire.com/video-sitemap-1.xml   2024-02-24 19:16:56   \n",
       "4  https://www.geekwire.com/video-sitemap-1.xml   2024-02-24 19:16:56   \n",
       "\n",
       "   sitemap_size_mb                    download_date  \n",
       "0         0.021546 2024-02-24 19:16:56.309302+00:00  \n",
       "1         0.021546 2024-02-24 19:16:56.309302+00:00  \n",
       "2         0.021546 2024-02-24 19:16:56.309302+00:00  \n",
       "3         0.021546 2024-02-24 19:16:56.309302+00:00  \n",
       "4         0.021546 2024-02-24 19:16:56.309302+00:00  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "geekWire = adv.sitemap_to_df('https://www.geekwire.com/sitemap.xml', recursive=True)\n",
    "\n",
    "geekWire.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium.webdriver.chrome.options import Options\n",
    "chrome_options = Options()\n",
    "chrome_options.add_argument(\"--headless=new\")\n",
    "\n",
    "# Path to your webdriver executable\n",
    "driver_path = '/Users/rmadith/Downloads/chromedriver_mac_arm64/chromedriver'\n",
    "\n",
    "# Initialize the WebDriver (example with Chrome)\n",
    "driver = webdriver.Chrome(options=chrome_options)\n",
    "\n",
    "url = geekWire.loc[0, 'loc']\n",
    "print(url)\n",
    "\n",
    "# Open the webpage\n",
    "driver.get(url)\n",
    "\n",
    "try:\n",
    "    # Wait for a specific element that indicates the page has loaded\n",
    "    # This needs to be adjusted based on the actual page structure\n",
    "    element = WebDriverWait(driver, 10).until(\n",
    "        EC.presence_of_element_located((By.CLASS_NAME, \"yoast-schema-graph\"))\n",
    "    )\n",
    "    \n",
    "    # Now you can access page_source or interact with the page\n",
    "    html_content = driver.page_source\n",
    "    #print(html_content)\n",
    "    \n",
    "    # You can now use BeautifulSoup or other methods to parse html_content\n",
    "finally:\n",
    "    driver.quit()\n",
    "\n",
    "# Parse the html content\n",
    "soup = BeautifulSoup(html_content, 'html.parser')\n",
    "\n",
    "# Get the title of the page\n",
    "title = soup.title.text\n",
    "\n",
    "# Get all <p> tags\n",
    "content = soup.find_all('p')\n",
    "\n",
    "# Remove all tags\n",
    "content = [c.text for c in content]\n",
    "\n",
    "content = ' '.join(content)\n",
    "\n",
    "title, content\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scrapping Wired "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-24 18:46:11,959 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml\n",
      "2024-02-24 18:46:12,029 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2024&month=2&week=4\n",
      "2024-02-24 18:46:12,087 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2024&month=2&week=3\n",
      "2024-02-24 18:46:12,147 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2024&month=2&week=2\n",
      "2024-02-24 18:46:14,294 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2024&month=2&week=1\n",
      "2024-02-24 18:46:14,355 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2024&month=1&week=5\n",
      "2024-02-24 18:46:14,414 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2024&month=1&week=4\n",
      "2024-02-24 18:46:14,474 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2024&month=1&week=3\n",
      "2024-02-24 18:46:14,532 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2024&month=1&week=2\n",
      "2024-02-24 18:46:14,588 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2024&month=1&week=1\n",
      "2024-02-24 18:46:14,647 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2023&month=12&week=5\n",
      "2024-02-24 18:46:14,704 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2023&month=12&week=4\n",
      "2024-02-24 18:46:14,762 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2023&month=12&week=3\n",
      "2024-02-24 18:46:14,821 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2023&month=12&week=2\n",
      "2024-02-24 18:46:14,878 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2023&month=12&week=1\n",
      "2024-02-24 18:46:14,936 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.wired.com/sitemap.xml?year=2023&month=11&week=5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loc</th>\n",
       "      <th>lastmod</th>\n",
       "      <th>sitemap</th>\n",
       "      <th>sitemap_last_modified</th>\n",
       "      <th>sitemap_size_mb</th>\n",
       "      <th>download_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.wired.com/gallery/best-massage-guns/</td>\n",
       "      <td>2024-02-24 16:00:00+00:00</td>\n",
       "      <td>https://www.wired.com/sitemap.xml?year=2024&amp;mo...</td>\n",
       "      <td>2024-02-24 20:15:16</td>\n",
       "      <td>0.005531</td>\n",
       "      <td>2024-02-25 00:46:12.032728+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.wired.com/story/apple-vision-pro-c...</td>\n",
       "      <td>2024-02-24 14:30:00+00:00</td>\n",
       "      <td>https://www.wired.com/sitemap.xml?year=2024&amp;mo...</td>\n",
       "      <td>2024-02-24 20:15:16</td>\n",
       "      <td>0.005531</td>\n",
       "      <td>2024-02-25 00:46:12.032728+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.wired.com/story/isoon-china-hack-f...</td>\n",
       "      <td>2024-02-24 14:00:00+00:00</td>\n",
       "      <td>https://www.wired.com/sitemap.xml?year=2024&amp;mo...</td>\n",
       "      <td>2024-02-24 20:15:16</td>\n",
       "      <td>0.005531</td>\n",
       "      <td>2024-02-25 00:46:12.032728+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.wired.com/story/climate-change-rai...</td>\n",
       "      <td>2024-02-24 13:00:00+00:00</td>\n",
       "      <td>https://www.wired.com/sitemap.xml?year=2024&amp;mo...</td>\n",
       "      <td>2024-02-24 20:15:16</td>\n",
       "      <td>0.005531</td>\n",
       "      <td>2024-02-25 00:46:12.032728+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.wired.com/story/8-sweet-deals-on-s...</td>\n",
       "      <td>2024-02-24 12:00:00+00:00</td>\n",
       "      <td>https://www.wired.com/sitemap.xml?year=2024&amp;mo...</td>\n",
       "      <td>2024-02-24 20:15:16</td>\n",
       "      <td>0.005531</td>\n",
       "      <td>2024-02-25 00:46:12.032728+00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 loc  \\\n",
       "0   https://www.wired.com/gallery/best-massage-guns/   \n",
       "1  https://www.wired.com/story/apple-vision-pro-c...   \n",
       "2  https://www.wired.com/story/isoon-china-hack-f...   \n",
       "3  https://www.wired.com/story/climate-change-rai...   \n",
       "4  https://www.wired.com/story/8-sweet-deals-on-s...   \n",
       "\n",
       "                    lastmod  \\\n",
       "0 2024-02-24 16:00:00+00:00   \n",
       "1 2024-02-24 14:30:00+00:00   \n",
       "2 2024-02-24 14:00:00+00:00   \n",
       "3 2024-02-24 13:00:00+00:00   \n",
       "4 2024-02-24 12:00:00+00:00   \n",
       "\n",
       "                                             sitemap sitemap_last_modified  \\\n",
       "0  https://www.wired.com/sitemap.xml?year=2024&mo...   2024-02-24 20:15:16   \n",
       "1  https://www.wired.com/sitemap.xml?year=2024&mo...   2024-02-24 20:15:16   \n",
       "2  https://www.wired.com/sitemap.xml?year=2024&mo...   2024-02-24 20:15:16   \n",
       "3  https://www.wired.com/sitemap.xml?year=2024&mo...   2024-02-24 20:15:16   \n",
       "4  https://www.wired.com/sitemap.xml?year=2024&mo...   2024-02-24 20:15:16   \n",
       "\n",
       "   sitemap_size_mb                    download_date  \n",
       "0         0.005531 2024-02-25 00:46:12.032728+00:00  \n",
       "1         0.005531 2024-02-25 00:46:12.032728+00:00  \n",
       "2         0.005531 2024-02-25 00:46:12.032728+00:00  \n",
       "3         0.005531 2024-02-25 00:46:12.032728+00:00  \n",
       "4         0.005531 2024-02-25 00:46:12.032728+00:00  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wired_months = adv.sitemap_to_df('https://www.wired.com/sitemap.xml', recursive=False)\n",
    "\n",
    "wired = []\n",
    "\n",
    "for i in range(0, 15):\n",
    "    url = wired_months.loc[i, 'loc']\n",
    "    wired.append(adv.sitemap_to_df(url, recursive=True))\n",
    "\n",
    "wired[0].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.wired.com/gallery/best-massage-guns/\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('7 Best Theragun-Alternative Massage Guns (2024): Portable, Affordable, and Heat Therapy | WIRED',\n",
       " 'If you want a massage gun but don’t want to pay Theragun prices—or maybe you just want different features and attachments—then get one of these picks for soothing sore muscles. +++lead-in-text\\nMassage guns, also known as percussive therapy devices, help relax and soothe sore muscles and decrease recovery time, whether you’re an athlete or just hunching over a desk all day. Some even include heat and cold therapy. We’ve tested several across price ranges, and here you’ll find our favorites, plus some fun extras like a massaging pillow and goggles.')"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get content of one url\n",
    "url = wired[0].loc[0, 'loc']\n",
    "\n",
    "# Get the content of the url\n",
    "response = requests.get(url)\n",
    "print(url)\n",
    "html = response.content\n",
    "\n",
    "# Parse the html content\n",
    "soup = BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "# Get the title of the page\n",
    "title = soup.title.text\n",
    "\n",
    "# Get the content of the page\n",
    "content = soup.find('script', type = 'application/ld+json').string\n",
    "\n",
    "data = json.loads(content)\n",
    "\n",
    "article_body = data.get(\"articleBody\", \"Text not found\")\n",
    "\n",
    "title, article_body\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "740"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lenght = 0\n",
    "\n",
    "for i in range(len(wired)):\n",
    "    lenght += len(wired[i])\n",
    "\n",
    "lenght"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add the content to the collection\n",
    "for i in range(len(wired)):\n",
    "    for j in range(len(wired[i])):\n",
    "        url = wired[i].loc[j, 'loc']\n",
    "        response = requests.get(url)\n",
    "        html = response.content\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "        title = soup.title.text\n",
    "        content = soup.find('script', type = 'application/ld+json')\n",
    "        if content is None:\n",
    "            continue\n",
    "        data = json.loads(content.string)\n",
    "        article_body = data.get(\"articleBody\", \"Text not found\")\n",
    "        if content is None or article_body is None or len(article_body) < 900:\n",
    "            continue\n",
    "        collection.add(\n",
    "            ids=url,\n",
    "            documents=article_body[:900],\n",
    "            metadatas={\n",
    "                \"title\": title,\n",
    "                \"url\": url,\n",
    "                \"source\": \"Wired\",\n",
    "            }\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scrapping Gizmodo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-24 18:57:26,793 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://gizmodo.com/sitemap_bydate.xml?startTime=2022-02-18T00:00:00&endTime=2024-02-24T23:59:59\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loc</th>\n",
       "      <th>lastmod</th>\n",
       "      <th>changefreq</th>\n",
       "      <th>sitemap</th>\n",
       "      <th>etag</th>\n",
       "      <th>sitemap_size_mb</th>\n",
       "      <th>download_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://gizmodo.com</td>\n",
       "      <td>2024-02-25 00:57:21+00:00</td>\n",
       "      <td>hourly</td>\n",
       "      <td>https://gizmodo.com/sitemap_bydate.xml?startTi...</td>\n",
       "      <td>W/\"2d185-jxy7z9qyDfMXPLnmKaK0MrjcyuA\"</td>\n",
       "      <td>0.176152</td>\n",
       "      <td>2024-02-25 00:57:26.802469+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://gizmodo.com/kane-chronicles-movies-net...</td>\n",
       "      <td>2024-02-24 21:15:00+00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://gizmodo.com/sitemap_bydate.xml?startTi...</td>\n",
       "      <td>W/\"2d185-jxy7z9qyDfMXPLnmKaK0MrjcyuA\"</td>\n",
       "      <td>0.176152</td>\n",
       "      <td>2024-02-25 00:57:26.802469+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://gizmodo.com/fast-xi-vin-diesel-status-...</td>\n",
       "      <td>2024-02-24 20:35:00+00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://gizmodo.com/sitemap_bydate.xml?startTi...</td>\n",
       "      <td>W/\"2d185-jxy7z9qyDfMXPLnmKaK0MrjcyuA\"</td>\n",
       "      <td>0.176152</td>\n",
       "      <td>2024-02-25 00:57:26.802469+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://gizmodo.com/elon-musks-latest-robot-vi...</td>\n",
       "      <td>2024-02-24 20:58:44+00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://gizmodo.com/sitemap_bydate.xml?startTi...</td>\n",
       "      <td>W/\"2d185-jxy7z9qyDfMXPLnmKaK0MrjcyuA\"</td>\n",
       "      <td>0.176152</td>\n",
       "      <td>2024-02-25 00:57:26.802469+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://gizmodo.com/jurassic-league-animated-m...</td>\n",
       "      <td>2024-02-24 19:05:00+00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://gizmodo.com/sitemap_bydate.xml?startTi...</td>\n",
       "      <td>W/\"2d185-jxy7z9qyDfMXPLnmKaK0MrjcyuA\"</td>\n",
       "      <td>0.176152</td>\n",
       "      <td>2024-02-25 00:57:26.802469+00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 loc  \\\n",
       "0                                https://gizmodo.com   \n",
       "1  https://gizmodo.com/kane-chronicles-movies-net...   \n",
       "2  https://gizmodo.com/fast-xi-vin-diesel-status-...   \n",
       "3  https://gizmodo.com/elon-musks-latest-robot-vi...   \n",
       "4  https://gizmodo.com/jurassic-league-animated-m...   \n",
       "\n",
       "                    lastmod changefreq  \\\n",
       "0 2024-02-25 00:57:21+00:00     hourly   \n",
       "1 2024-02-24 21:15:00+00:00        NaN   \n",
       "2 2024-02-24 20:35:00+00:00        NaN   \n",
       "3 2024-02-24 20:58:44+00:00        NaN   \n",
       "4 2024-02-24 19:05:00+00:00        NaN   \n",
       "\n",
       "                                             sitemap  \\\n",
       "0  https://gizmodo.com/sitemap_bydate.xml?startTi...   \n",
       "1  https://gizmodo.com/sitemap_bydate.xml?startTi...   \n",
       "2  https://gizmodo.com/sitemap_bydate.xml?startTi...   \n",
       "3  https://gizmodo.com/sitemap_bydate.xml?startTi...   \n",
       "4  https://gizmodo.com/sitemap_bydate.xml?startTi...   \n",
       "\n",
       "                                    etag  sitemap_size_mb  \\\n",
       "0  W/\"2d185-jxy7z9qyDfMXPLnmKaK0MrjcyuA\"         0.176152   \n",
       "1  W/\"2d185-jxy7z9qyDfMXPLnmKaK0MrjcyuA\"         0.176152   \n",
       "2  W/\"2d185-jxy7z9qyDfMXPLnmKaK0MrjcyuA\"         0.176152   \n",
       "3  W/\"2d185-jxy7z9qyDfMXPLnmKaK0MrjcyuA\"         0.176152   \n",
       "4  W/\"2d185-jxy7z9qyDfMXPLnmKaK0MrjcyuA\"         0.176152   \n",
       "\n",
       "                     download_date  \n",
       "0 2024-02-25 00:57:26.802469+00:00  \n",
       "1 2024-02-25 00:57:26.802469+00:00  \n",
       "2 2024-02-25 00:57:26.802469+00:00  \n",
       "3 2024-02-25 00:57:26.802469+00:00  \n",
       "4 2024-02-25 00:57:26.802469+00:00  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gizmodo = adv.sitemap_to_df(f'https://gizmodo.com/sitemap_bydate.xml?startTime=2022-02-18T00:00:00&endTime=2024-02-24T23:59:59', recursive=True)\n",
    "\n",
    "gizmodo.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('elon musk optimus tesla robot blurry cellphone 2002',\n",
       " 'Elon Musk has shared a new video on Saturday featuring Optimus, the robot Tesla has been working on since 2021. But anyone who tries to watch the video will immediately notice something weird. The clip of Optimus is so low quality and pixelated that it looks like it was shot on a flip-phone from two decades ago.\\n\\n\\nThe new video was posted in the early morning hours of Saturday and has been viewed over 35 million times as of this writing. But the video appears to show Optimus just walking around without doing much of anything. That would have been quite impressive around 2013 or so, since it’s relatively difficult to get machines to walk like humans, but it’s not entirely clear why Musk would want the world to see Optimus walking like this.\\n\\nIncremental technical achievements aside, why does this video look so terrible? We weren’t the only ones to notice the bizarrely pixelated quality, as plenty of Musk fans made jokes about the blurriness.\\n\\n“Was this filmed with a potato?” one user quipped.\\n\\n“Same photographer?” another X user quipped with a photo of Bigfoot.\\n\\n\\nTesla didn’t immediately respond to questions about this new video of Optimus emailed Saturday.\\n\\nMusk unveiled Optimus with an unconventional presentation in the summer of 2021 that really felt like the billionaire was desperate to hype virtually anything futuristic. Tesla’s AI Day that year didn’t feature a real robot, but rather someone dressed in a white and black suit moving around like a stereotypical robot before starting to dance a jig.\\n\\nTesla’s robot has made progress since that first jokey unveiling, but Optimus still has quite a ways to go before it can catch up to the most cutting edge robots of the 2020s. Atlas, a humanoid robot made by Boston Dynamics, started learning how to pick itself up in 2016, standing on one leg that same year, doing backflips in 2017, and achieved parkour-style jumping in 2018. \\n\\nAnd Atlas is still making progress in ways that rival how humans actually move. Last year, the Atlas robot showed off its ability to manipulate its environment to navigate complex worksites.\\n\\n\\nOptimus has made improvements since it was first announced but it has quite a ways to go if it wants to catch up to a company like Boston Dynamics. Arguably the most impressive thing we’ve seen Optimus do is fold laundry, but if you take a close look at the video, there was a person standing just off-screen mimicking the movements. And, frankly, that’s technology that’s been possible since the 1960s.\\n\\nCan Tesla develop a truly autonomous robot that can work as a household servant, just as Musk has promised? Only time will tell. But we’ve been waiting on that version of the future for over a century now. Robotics is hard. But we can certainly keep dreaming.\\n\\n\\n')"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get content of one url\n",
    "url = gizmodo.loc[1, 'loc']\n",
    "\n",
    "# Get the content of the url\n",
    "response = requests.get(url)\n",
    "\n",
    "html = response.content\n",
    "\n",
    "# Parse the html content\n",
    "soup = BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "# Get the title of the page\n",
    "title = soup.title.text\n",
    "\n",
    "# Get the content of the page\n",
    "content = soup.find('script', type = 'application/ld+json').string\n",
    "\n",
    "data = json.loads(content)\n",
    "\n",
    "article_body = data.get(\"articleBody\", \"Text not found\")\n",
    "\n",
    "title, article_body"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(gizmodo)):\n",
    "    url = gizmodo.loc[i, 'loc']\n",
    "    response = requests.get(url)\n",
    "    html = response.content\n",
    "    soup = BeautifulSoup(html, 'html.parser')\n",
    "    title = soup.title.text\n",
    "    content = soup.find('script', type = 'application/ld+json')\n",
    "    if content is None:\n",
    "        continue\n",
    "    data = json.loads(content.string)\n",
    "    article_body = data.get(\"articleBody\", \"Text not found\")\n",
    "    if content is None or article_body is None or len(article_body) < 900:\n",
    "        continue\n",
    "    collection.add(\n",
    "        ids=url,\n",
    "        documents=article_body[:900],\n",
    "        metadatas={\n",
    "            \"title\": title,\n",
    "            \"url\": url,\n",
    "            \"source\": \"Gizmodo\",\n",
    "        }\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cnet Scrapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-24 19:11:54,179 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2023/09.xml\n",
      "2024-02-24 19:11:54,185 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2023/08.xml\n",
      "2024-02-24 19:11:54,330 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2024/01.xml\n",
      "2024-02-24 19:11:54,684 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2023/11.xml\n",
      "2024-02-24 19:11:55,043 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2023/10.xml\n",
      "2024-02-24 19:11:55,169 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2023/12.xml\n",
      "2024-02-24 19:11:55,743 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2023/04.xml\n",
      "2024-02-24 19:11:55,822 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2023/07.xml\n",
      "2024-02-24 19:11:55,845 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2024/02.xml\n",
      "2024-02-24 19:11:56,445 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2023/06.xml\n",
      "2024-02-24 19:11:56,625 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2023/05.xml\n",
      "2024-02-24 19:11:58,004 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2023/03.xml\n",
      "2024-02-24 19:11:58,649 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2023/02.xml\n",
      "2024-02-24 19:11:59,422 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2022/12.xml\n",
      "2024-02-24 19:12:00,262 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2022/09.xml\n",
      "2024-02-24 19:12:00,284 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2022/08.xml\n",
      "2024-02-24 19:12:01,117 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2022/11.xml\n",
      "2024-02-24 19:12:02,279 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2023/01.xml\n",
      "2024-02-24 19:12:02,298 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2022/10.xml\n",
      "2024-02-24 19:12:03,073 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2022/06.xml\n",
      "2024-02-24 19:12:03,443 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2022/03.xml\n",
      "2024-02-24 19:12:04,141 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2022/02.xml\n",
      "2024-02-24 19:12:04,424 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2022/04.xml\n",
      "2024-02-24 19:12:04,687 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2022/05.xml\n",
      "2024-02-24 19:12:06,227 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2021/12.xml\n",
      "2024-02-24 19:12:06,925 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2022/01.xml\n",
      "2024-02-24 19:12:06,930 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2021/11.xml\n",
      "2024-02-24 19:12:07,304 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2021/08.xml\n",
      "2024-02-24 19:12:07,847 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2021/09.xml\n",
      "2024-02-24 19:12:08,374 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2021/10.xml\n",
      "2024-02-24 19:12:09,133 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2022/07.xml\n",
      "2024-02-24 19:12:10,001 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2021/04.xml\n",
      "2024-02-24 19:12:10,027 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2021/07.xml\n",
      "2024-02-24 19:12:10,999 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2021/05.xml\n",
      "2024-02-24 19:12:11,160 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2021/02.xml\n",
      "2024-02-24 19:12:11,299 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2021/03.xml\n",
      "2024-02-24 19:12:11,376 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2021/01.xml\n",
      "2024-02-24 19:12:12,524 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2020/12.xml\n",
      "2024-02-24 19:12:12,672 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2021/06.xml\n",
      "2024-02-24 19:12:13,363 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2020/11.xml\n",
      "2024-02-24 19:12:13,805 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2020/10.xml\n",
      "2024-02-24 19:12:14,083 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2020/07.xml\n",
      "2024-02-24 19:12:14,348 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2020/06.xml\n",
      "2024-02-24 19:12:14,589 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2020/08.xml\n",
      "2024-02-24 19:12:14,676 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2020/09.xml\n",
      "2024-02-24 19:12:16,499 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2020/05.xml\n",
      "2024-02-24 19:12:16,771 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2020/03.xml\n",
      "2024-02-24 19:12:17,228 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2019/12.xml\n",
      "2024-02-24 19:12:17,255 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2020/04.xml\n",
      "2024-02-24 19:12:17,418 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2019/10.xml\n",
      "2024-02-24 19:12:17,749 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2020/01.xml\n",
      "2024-02-24 19:12:17,983 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2020/02.xml\n",
      "2024-02-24 19:12:21,061 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2019/08.xml\n",
      "2024-02-24 19:12:21,092 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2019/11.xml\n",
      "2024-02-24 19:12:21,662 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2019/06.xml\n",
      "2024-02-24 19:12:22,128 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2019/04.xml\n",
      "2024-02-24 19:12:22,550 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2019/05.xml\n",
      "2024-02-24 19:12:22,576 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2019/07.xml\n",
      "2024-02-24 19:12:23,035 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2019/03.xml\n",
      "2024-02-24 19:12:23,103 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2019/09.xml\n",
      "2024-02-24 19:12:24,884 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2018/12.xml\n",
      "2024-02-24 19:12:25,684 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2019/02.xml\n",
      "2024-02-24 19:12:25,858 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2019/01.xml\n",
      "2024-02-24 19:12:25,890 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2018/11.xml\n",
      "2024-02-24 19:12:26,214 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2018/09.xml\n",
      "2024-02-24 19:12:26,434 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2018/08.xml\n",
      "2024-02-24 19:12:27,231 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2018/07.xml\n",
      "2024-02-24 19:12:28,310 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2017/12.xml\n",
      "2024-02-24 19:12:29,874 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2018/01.xml\n",
      "2024-02-24 19:12:30,704 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2017/11.xml\n",
      "2024-02-24 19:12:30,962 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2018/10.xml\n",
      "2024-02-24 19:12:31,700 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2018/04.xml\n",
      "2024-02-24 19:12:31,940 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2018/02.xml\n",
      "2024-02-24 19:12:32,234 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2018/05.xml\n",
      "2024-02-24 19:12:32,523 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2017/09.xml\n",
      "2024-02-24 19:12:33,047 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2018/03.xml\n",
      "2024-02-24 19:12:33,124 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2018/06.xml\n",
      "2024-02-24 19:12:33,156 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2017/08.xml\n",
      "2024-02-24 19:12:33,196 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2017/10.xml\n",
      "2024-02-24 19:12:35,053 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2017/07.xml\n",
      "2024-02-24 19:12:35,077 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2017/04.xml\n",
      "2024-02-24 19:12:35,107 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2017/05.xml\n",
      "2024-02-24 19:12:35,622 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2016/12.xml\n",
      "2024-02-24 19:12:35,928 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2017/02.xml\n",
      "2024-02-24 19:12:35,957 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2017/01.xml\n",
      "2024-02-24 19:12:37,835 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2017/03.xml\n",
      "2024-02-24 19:12:37,863 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2016/09.xml\n",
      "2024-02-24 19:12:39,245 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2017/06.xml\n",
      "2024-02-24 19:12:39,421 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2016/06.xml\n",
      "2024-02-24 19:12:39,804 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2016/11.xml\n",
      "2024-02-24 19:12:39,826 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2016/10.xml\n",
      "2024-02-24 19:12:39,889 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2016/04.xml\n",
      "2024-02-24 19:12:39,998 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2016/08.xml\n",
      "2024-02-24 19:12:40,418 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2016/05.xml\n",
      "2024-02-24 19:12:41,614 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2016/03.xml\n",
      "2024-02-24 19:12:42,076 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2015/11.xml\n",
      "2024-02-24 19:12:42,405 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2015/12.xml\n",
      "2024-02-24 19:12:42,507 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2015/10.xml\n",
      "2024-02-24 19:12:42,696 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2016/01.xml\n",
      "2024-02-24 19:12:43,473 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2016/02.xml\n",
      "2024-02-24 19:12:43,613 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2015/09.xml\n",
      "2024-02-24 19:12:43,718 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2015/08.xml\n",
      "2024-02-24 19:12:44,339 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2016/07.xml\n",
      "2024-02-24 19:12:44,477 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2015/04.xml\n",
      "2024-02-24 19:12:44,683 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2015/07.xml\n",
      "2024-02-24 19:12:45,623 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2015/05.xml\n",
      "2024-02-24 19:12:45,635 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2015/06.xml\n",
      "2024-02-24 19:12:45,735 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2015/01.xml\n",
      "2024-02-24 19:12:46,445 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2014/12.xml\n",
      "2024-02-24 19:12:46,852 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2014/10.xml\n",
      "2024-02-24 19:12:47,123 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2015/02.xml\n",
      "2024-02-24 19:12:47,682 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2014/11.xml\n",
      "2024-02-24 19:12:48,193 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2014/09.xml\n",
      "2024-02-24 19:12:50,880 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2014/05.xml\n",
      "2024-02-24 19:12:51,069 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2014/06.xml\n",
      "2024-02-24 19:12:51,314 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2015/03.xml\n",
      "2024-02-24 19:12:51,821 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2014/03.xml\n",
      "2024-02-24 19:12:51,866 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2014/02.xml\n",
      "2024-02-24 19:12:52,040 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2014/08.xml\n",
      "2024-02-24 19:12:52,068 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2014/07.xml\n",
      "2024-02-24 19:12:52,362 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2014/04.xml\n",
      "2024-02-24 19:12:54,513 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2013/11.xml\n",
      "2024-02-24 19:12:54,678 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2013/08.xml\n",
      "2024-02-24 19:12:55,181 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2014/01.xml\n",
      "2024-02-24 19:12:55,356 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2013/10.xml\n",
      "2024-02-24 19:12:56,195 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2013/09.xml\n",
      "2024-02-24 19:12:56,803 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2013/12.xml\n",
      "2024-02-24 19:12:57,107 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2013/05.xml\n",
      "2024-02-24 19:12:57,492 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2013/06.xml\n",
      "2024-02-24 19:12:57,755 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2013/07.xml\n",
      "2024-02-24 19:12:58,677 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2013/04.xml\n",
      "2024-02-24 19:12:59,271 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2013/03.xml\n",
      "2024-02-24 19:13:00,277 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2013/02.xml\n",
      "2024-02-24 19:13:01,298 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2012/11.xml\n",
      "2024-02-24 19:13:02,838 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2013/01.xml\n",
      "2024-02-24 19:13:02,850 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2012/09.xml\n",
      "2024-02-24 19:13:03,936 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2012/07.xml\n",
      "2024-02-24 19:13:03,985 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2012/10.xml\n",
      "2024-02-24 19:13:04,994 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2012/12.xml\n",
      "2024-02-24 19:13:05,069 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2012/06.xml\n",
      "2024-02-24 19:13:05,131 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2012/05.xml\n",
      "2024-02-24 19:13:05,185 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2012/08.xml\n",
      "2024-02-24 19:13:07,280 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2012/02.xml\n",
      "2024-02-24 19:13:07,573 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2012/04.xml\n",
      "2024-02-24 19:13:07,866 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2012/03.xml\n",
      "2024-02-24 19:13:08,384 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2011/12.xml\n",
      "2024-02-24 19:13:08,432 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2012/01.xml\n",
      "2024-02-24 19:13:08,756 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2011/10.xml\n",
      "2024-02-24 19:13:09,421 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2011/11.xml\n",
      "2024-02-24 19:13:09,923 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2011/09.xml\n",
      "2024-02-24 19:13:10,821 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2011/08.xml\n",
      "2024-02-24 19:13:11,890 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2011/05.xml\n",
      "2024-02-24 19:13:12,991 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2011/03.xml\n",
      "2024-02-24 19:13:13,351 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2011/04.xml\n",
      "2024-02-24 19:13:13,392 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2011/02.xml\n",
      "2024-02-24 19:13:13,691 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2010/12.xml\n",
      "2024-02-24 19:13:14,918 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2010/11.xml\n",
      "2024-02-24 19:13:15,970 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2011/01.xml\n",
      "2024-02-24 19:13:16,351 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2010/10.xml\n",
      "2024-02-24 19:13:16,477 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2010/09.xml\n",
      "2024-02-24 19:13:16,583 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2010/08.xml\n",
      "2024-02-24 19:13:17,680 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2010/07.xml\n",
      "2024-02-24 19:13:17,774 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2011/06.xml\n",
      "2024-02-24 19:13:17,774 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2011/07.xml\n",
      "2024-02-24 19:13:18,740 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2010/05.xml\n",
      "2024-02-24 19:13:19,097 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2010/06.xml\n",
      "2024-02-24 19:13:19,743 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2010/04.xml\n",
      "2024-02-24 19:13:19,787 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2010/01.xml\n",
      "2024-02-24 19:13:19,833 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2010/02.xml\n",
      "2024-02-24 19:13:20,838 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2010/03.xml\n",
      "2024-02-24 19:13:22,463 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2009/08.xml\n",
      "2024-02-24 19:13:23,148 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2009/10.xml\n",
      "2024-02-24 19:13:23,529 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2009/12.xml\n",
      "2024-02-24 19:13:23,607 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2009/07.xml\n",
      "2024-02-24 19:13:23,832 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2009/05.xml\n",
      "2024-02-24 19:13:23,904 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2009/11.xml\n",
      "2024-02-24 19:13:24,448 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2009/06.xml\n",
      "2024-02-24 19:13:26,236 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2009/04.xml\n",
      "2024-02-24 19:13:27,704 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2008/11.xml\n",
      "2024-02-24 19:13:28,034 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2008/12.xml\n",
      "2024-02-24 19:13:28,662 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2008/10.xml\n",
      "2024-02-24 19:13:28,798 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2009/02.xml\n",
      "2024-02-24 19:13:29,387 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2009/01.xml\n",
      "2024-02-24 19:13:30,290 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2009/03.xml\n",
      "2024-02-24 19:13:31,277 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2008/08.xml\n",
      "2024-02-24 19:13:32,216 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2008/06.xml\n",
      "2024-02-24 19:13:32,329 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2008/05.xml\n",
      "2024-02-24 19:13:32,781 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2008/04.xml\n",
      "2024-02-24 19:13:32,860 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2008/09.xml\n",
      "2024-02-24 19:13:33,971 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2008/07.xml\n",
      "2024-02-24 19:13:34,154 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2008/03.xml\n",
      "2024-02-24 19:13:35,395 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2007/11.xml\n",
      "2024-02-24 19:13:35,495 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2007/12.xml\n",
      "2024-02-24 19:13:37,806 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2007/09.xml\n",
      "2024-02-24 19:13:37,814 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2007/08.xml\n",
      "2024-02-24 19:13:37,857 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2008/02.xml\n",
      "2024-02-24 19:13:38,097 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2008/01.xml\n",
      "2024-02-24 19:13:38,286 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2007/10.xml\n",
      "2024-02-24 19:13:38,352 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2007/06.xml\n",
      "2024-02-24 19:13:38,477 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2007/07.xml\n",
      "2024-02-24 19:13:39,276 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2007/03.xml\n",
      "2024-02-24 19:13:40,269 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2006/12.xml\n",
      "2024-02-24 19:13:40,734 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2007/05.xml\n",
      "2024-02-24 19:13:40,788 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2006/10.xml\n",
      "2024-02-24 19:13:41,628 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2007/01.xml\n",
      "2024-02-24 19:13:41,731 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2006/09.xml\n",
      "2024-02-24 19:13:41,743 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2007/04.xml\n",
      "2024-02-24 19:13:42,944 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2006/08.xml\n",
      "2024-02-24 19:13:43,004 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2006/07.xml\n",
      "2024-02-24 19:13:43,275 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2006/11.xml\n",
      "2024-02-24 19:13:43,565 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2006/06.xml\n",
      "2024-02-24 19:13:44,790 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2006/02.xml\n",
      "2024-02-24 19:13:44,797 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2006/04.xml\n",
      "2024-02-24 19:13:44,811 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2006/05.xml\n",
      "2024-02-24 19:13:44,880 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2006/03.xml\n",
      "2024-02-24 19:13:46,884 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2005/11.xml\n",
      "2024-02-24 19:13:46,907 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2006/01.xml\n",
      "2024-02-24 19:13:46,970 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2005/12.xml\n",
      "2024-02-24 19:13:48,587 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2005/06.xml\n",
      "2024-02-24 19:13:48,958 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2005/07.xml\n",
      "2024-02-24 19:13:48,969 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2005/05.xml\n",
      "2024-02-24 19:13:49,304 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2005/08.xml\n",
      "2024-02-24 19:13:49,586 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2005/04.xml\n",
      "2024-02-24 19:13:49,815 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 503: first byte timeout https://www.cnet.com/sitemaps/article/2009/09.xml\n",
      "2024-02-24 19:13:50,619 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2005/02.xml\n",
      "2024-02-24 19:13:50,945 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2005/03.xml\n",
      "2024-02-24 19:13:51,415 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2005/01.xml\n",
      "2024-02-24 19:13:52,160 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2004/11.xml\n",
      "2024-02-24 19:13:52,627 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2004/12.xml\n",
      "2024-02-24 19:13:53,444 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2004/08.xml\n",
      "2024-02-24 19:13:53,897 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2004/09.xml\n",
      "2024-02-24 19:13:54,422 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2004/07.xml\n",
      "2024-02-24 19:13:54,726 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2004/06.xml\n",
      "2024-02-24 19:13:55,649 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2004/04.xml\n",
      "2024-02-24 19:13:55,860 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2004/03.xml\n",
      "2024-02-24 19:13:56,105 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2004/05.xml\n",
      "2024-02-24 19:13:56,284 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2004/10.xml\n",
      "2024-02-24 19:13:57,382 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2004/02.xml\n",
      "2024-02-24 19:13:57,867 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2003/11.xml\n",
      "2024-02-24 19:13:58,591 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2003/12.xml\n",
      "2024-02-24 19:13:59,220 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2003/10.xml\n",
      "2024-02-24 19:13:59,456 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2003/09.xml\n",
      "2024-02-24 19:13:59,539 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2003/08.xml\n",
      "2024-02-24 19:14:00,016 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2003/07.xml\n",
      "2024-02-24 19:14:00,749 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2003/05.xml\n",
      "2024-02-24 19:14:02,118 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2003/06.xml\n",
      "2024-02-24 19:14:02,126 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2003/04.xml\n",
      "2024-02-24 19:14:02,370 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2003/03.xml\n",
      "2024-02-24 19:14:02,851 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2003/02.xml\n",
      "2024-02-24 19:14:04,138 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2002/12.xml\n",
      "2024-02-24 19:14:04,338 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2003/01.xml\n",
      "2024-02-24 19:14:04,567 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2002/11.xml\n",
      "2024-02-24 19:14:05,051 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2002/10.xml\n",
      "2024-02-24 19:14:05,204 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2002/08.xml\n",
      "2024-02-24 19:14:06,088 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2002/09.xml\n",
      "2024-02-24 19:14:06,527 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2002/07.xml\n",
      "2024-02-24 19:14:06,964 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2002/05.xml\n",
      "2024-02-24 19:14:07,071 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2002/06.xml\n",
      "2024-02-24 19:14:07,988 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2002/02.xml\n",
      "2024-02-24 19:14:08,644 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2002/01.xml\n",
      "2024-02-24 19:14:08,857 | WARNING | sitemaps.py:556 | sitemap_to_df | HTTP Error 503: first byte timeout https://www.cnet.com/sitemaps/article/2007/02.xml\n",
      "2024-02-24 19:14:08,879 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2002/03.xml\n",
      "2024-02-24 19:14:09,508 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2002/04.xml\n",
      "2024-02-24 19:14:09,589 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2001/12.xml\n",
      "2024-02-24 19:14:11,504 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2001/09.xml\n",
      "2024-02-24 19:14:11,510 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2001/10.xml\n",
      "2024-02-24 19:14:11,553 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2001/11.xml\n",
      "2024-02-24 19:14:11,722 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2001/08.xml\n",
      "2024-02-24 19:14:12,411 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2001/07.xml\n",
      "2024-02-24 19:14:12,901 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2004/01.xml\n",
      "2024-02-24 19:14:13,461 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2001/06.xml\n",
      "2024-02-24 19:14:14,645 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2001/05.xml\n",
      "2024-02-24 19:14:15,172 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2005/10.xml\n",
      "2024-02-24 19:14:15,682 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2001/03.xml\n",
      "2024-02-24 19:14:15,690 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2005/09.xml\n",
      "2024-02-24 19:14:15,987 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2000/11.xml\n",
      "2024-02-24 19:14:16,239 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2001/01.xml\n",
      "2024-02-24 19:14:16,313 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2001/02.xml\n",
      "2024-02-24 19:14:16,474 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2001/04.xml\n",
      "2024-02-24 19:14:16,848 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2000/12.xml\n",
      "2024-02-24 19:14:18,473 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2000/08.xml\n",
      "2024-02-24 19:14:18,484 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2000/06.xml\n",
      "2024-02-24 19:14:18,644 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2000/07.xml\n",
      "2024-02-24 19:14:18,897 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2000/09.xml\n",
      "2024-02-24 19:14:19,906 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2000/10.xml\n",
      "2024-02-24 19:14:20,039 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2000/03.xml\n",
      "2024-02-24 19:14:20,095 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2000/04.xml\n",
      "2024-02-24 19:14:20,111 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2000/05.xml\n",
      "2024-02-24 19:14:21,755 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2000/01.xml\n",
      "2024-02-24 19:14:22,302 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/2000/02.xml\n",
      "2024-02-24 19:14:22,523 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1999/10.xml\n",
      "2024-02-24 19:14:23,388 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1999/09.xml\n",
      "2024-02-24 19:14:23,394 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1999/07.xml\n",
      "2024-02-24 19:14:24,112 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1999/12.xml\n",
      "2024-02-24 19:14:24,113 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1999/04.xml\n",
      "2024-02-24 19:14:24,121 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1999/11.xml\n",
      "2024-02-24 19:14:24,234 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1999/01.xml\n",
      "2024-02-24 19:14:24,807 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1999/08.xml\n",
      "2024-02-24 19:14:24,988 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1999/05.xml\n",
      "2024-02-24 19:14:25,274 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1999/03.xml\n",
      "2024-02-24 19:14:25,298 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1999/06.xml\n",
      "2024-02-24 19:14:26,355 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1998/09.xml\n",
      "2024-02-24 19:14:26,465 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1998/10.xml\n",
      "2024-02-24 19:14:26,936 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1998/11.xml\n",
      "2024-02-24 19:14:27,332 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1998/08.xml\n",
      "2024-02-24 19:14:27,609 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1999/02.xml\n",
      "2024-02-24 19:14:27,766 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1998/05.xml\n",
      "2024-02-24 19:14:27,770 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1998/12.xml\n",
      "2024-02-24 19:14:28,307 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1998/07.xml\n",
      "2024-02-24 19:14:29,182 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1998/06.xml\n",
      "2024-02-24 19:14:29,417 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1998/03.xml\n",
      "2024-02-24 19:14:29,574 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1998/01.xml\n",
      "2024-02-24 19:14:29,939 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1998/02.xml\n",
      "2024-02-24 19:14:30,045 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1998/04.xml\n",
      "2024-02-24 19:14:30,197 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1997/11.xml\n",
      "2024-02-24 19:14:30,430 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1997/10.xml\n",
      "2024-02-24 19:14:31,180 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1997/08.xml\n",
      "2024-02-24 19:14:31,258 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1997/12.xml\n",
      "2024-02-24 19:14:31,423 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1997/09.xml\n",
      "2024-02-24 19:14:31,990 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1997/07.xml\n",
      "2024-02-24 19:14:33,016 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1997/04.xml\n",
      "2024-02-24 19:14:33,070 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1997/06.xml\n",
      "2024-02-24 19:14:33,083 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1997/05.xml\n",
      "2024-02-24 19:14:33,104 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1996/12.xml\n",
      "2024-02-24 19:14:33,381 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1997/03.xml\n",
      "2024-02-24 19:14:33,818 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1996/08.xml\n",
      "2024-02-24 19:14:34,423 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1996/05.xml\n",
      "2024-02-24 19:14:34,444 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1997/02.xml\n",
      "2024-02-24 19:14:34,548 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1996/07.xml\n",
      "2024-02-24 19:14:34,560 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1996/11.xml\n",
      "2024-02-24 19:14:34,583 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1996/10.xml\n",
      "2024-02-24 19:14:35,141 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1996/01.xml\n",
      "2024-02-24 19:14:35,322 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1996/03.xml\n",
      "2024-02-24 19:14:35,448 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1995/10.xml\n",
      "2024-02-24 19:14:35,480 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1997/01.xml\n",
      "2024-02-24 19:14:35,595 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1995/12.xml\n",
      "2024-02-24 19:14:35,622 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1996/04.xml\n",
      "2024-02-24 19:14:36,164 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1996/02.xml\n",
      "2024-02-24 19:14:36,782 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1996/06.xml\n",
      "2024-02-24 19:14:36,798 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1996/09.xml\n",
      "2024-02-24 19:14:37,190 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1995/06.xml\n",
      "2024-02-24 19:14:37,286 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1995/07.xml\n",
      "2024-02-24 19:14:37,318 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1995/11.xml\n",
      "2024-02-24 19:14:37,392 | INFO | sitemaps.py:563 | sitemap_to_df | Getting https://www.cnet.com/sitemaps/article/1995/09.xml\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loc</th>\n",
       "      <th>lastmod</th>\n",
       "      <th>image_loc</th>\n",
       "      <th>sitemap</th>\n",
       "      <th>sitemap_size_mb</th>\n",
       "      <th>download_date</th>\n",
       "      <th>errors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.cnet.com/deals/get-a-1-year-subscr...</td>\n",
       "      <td>2023-09-30 19:09:22+00:00</td>\n",
       "      <td>https://www.cnet.com/a/img/resize/26a3c506002e...</td>\n",
       "      <td>https://www.cnet.com/sitemaps/article/2023/09.xml</td>\n",
       "      <td>0.31424</td>\n",
       "      <td>2024-02-25 01:11:54.195104+00:00</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.cnet.com/deals/snag-a-4th-gen-echo...</td>\n",
       "      <td>2023-09-30 18:58:56+00:00</td>\n",
       "      <td>https://www.cnet.com/a/img/resize/e890cf4e2535...</td>\n",
       "      <td>https://www.cnet.com/sitemaps/article/2023/09.xml</td>\n",
       "      <td>0.31424</td>\n",
       "      <td>2024-02-25 01:11:54.195104+00:00</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.cnet.com/tech/gaming/should-you-bu...</td>\n",
       "      <td>2023-10-01 01:00:14+00:00</td>\n",
       "      <td>https://www.cnet.com/a/img/resize/cde106b2d3f1...</td>\n",
       "      <td>https://www.cnet.com/sitemaps/article/2023/09.xml</td>\n",
       "      <td>0.31424</td>\n",
       "      <td>2024-02-25 01:11:54.195104+00:00</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.cnet.com/tech/mobile/samsung-galax...</td>\n",
       "      <td>2023-09-30 01:02:26+00:00</td>\n",
       "      <td>https://www.cnet.com/a/img/resize/109e1f9d557a...</td>\n",
       "      <td>https://www.cnet.com/sitemaps/article/2023/09.xml</td>\n",
       "      <td>0.31424</td>\n",
       "      <td>2024-02-25 01:11:54.195104+00:00</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.cnet.com/tech/computing/nerf-launc...</td>\n",
       "      <td>2023-09-29 23:02:34+00:00</td>\n",
       "      <td>https://www.cnet.com/a/img/resize/550bd373c81a...</td>\n",
       "      <td>https://www.cnet.com/sitemaps/article/2023/09.xml</td>\n",
       "      <td>0.31424</td>\n",
       "      <td>2024-02-25 01:11:54.195104+00:00</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 loc  \\\n",
       "0  https://www.cnet.com/deals/get-a-1-year-subscr...   \n",
       "1  https://www.cnet.com/deals/snag-a-4th-gen-echo...   \n",
       "2  https://www.cnet.com/tech/gaming/should-you-bu...   \n",
       "3  https://www.cnet.com/tech/mobile/samsung-galax...   \n",
       "4  https://www.cnet.com/tech/computing/nerf-launc...   \n",
       "\n",
       "                    lastmod  \\\n",
       "0 2023-09-30 19:09:22+00:00   \n",
       "1 2023-09-30 18:58:56+00:00   \n",
       "2 2023-10-01 01:00:14+00:00   \n",
       "3 2023-09-30 01:02:26+00:00   \n",
       "4 2023-09-29 23:02:34+00:00   \n",
       "\n",
       "                                           image_loc  \\\n",
       "0  https://www.cnet.com/a/img/resize/26a3c506002e...   \n",
       "1  https://www.cnet.com/a/img/resize/e890cf4e2535...   \n",
       "2  https://www.cnet.com/a/img/resize/cde106b2d3f1...   \n",
       "3  https://www.cnet.com/a/img/resize/109e1f9d557a...   \n",
       "4  https://www.cnet.com/a/img/resize/550bd373c81a...   \n",
       "\n",
       "                                             sitemap  sitemap_size_mb  \\\n",
       "0  https://www.cnet.com/sitemaps/article/2023/09.xml          0.31424   \n",
       "1  https://www.cnet.com/sitemaps/article/2023/09.xml          0.31424   \n",
       "2  https://www.cnet.com/sitemaps/article/2023/09.xml          0.31424   \n",
       "3  https://www.cnet.com/sitemaps/article/2023/09.xml          0.31424   \n",
       "4  https://www.cnet.com/sitemaps/article/2023/09.xml          0.31424   \n",
       "\n",
       "                     download_date errors  \n",
       "0 2024-02-25 01:11:54.195104+00:00    NaN  \n",
       "1 2024-02-25 01:11:54.195104+00:00    NaN  \n",
       "2 2024-02-25 01:11:54.195104+00:00    NaN  \n",
       "3 2024-02-25 01:11:54.195104+00:00    NaN  \n",
       "4 2024-02-25 01:11:54.195104+00:00    NaN  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnet = adv.sitemap_to_df('https://www.cnet.com/sitemaps/article/index.xml', recursive=True)\n",
    "\n",
    "cnet.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.cnet.com/tech/computing/nerf-launches-its-own-sport-using-hit-detection-gear-and-lots-of-blasters/\n",
      "Text not found\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "string indices must be integers",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[30], line 32\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[38;5;28mprint\u001b[39m(article_body)\n\u001b[1;32m     31\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m article_body:\n\u001b[0;32m---> 32\u001b[0m     text\u001b[38;5;241m.\u001b[39mappend(\u001b[43mi\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43macceptedAnswer\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[1;32m     34\u001b[0m text \u001b[38;5;241m=\u001b[39m  \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(text)\n\u001b[1;32m     36\u001b[0m \u001b[38;5;66;03m# remove <p> tags\u001b[39;00m\n",
      "\u001b[0;31mTypeError\u001b[0m: string indices must be integers"
     ]
    }
   ],
   "source": [
    "# Get content of one url\n",
    "url = cnet.loc[4, 'loc']\n",
    "\n",
    "# Get the content of the url\n",
    "response = requests.get(url)\n",
    "print(url)\n",
    "\n",
    "html = response.content\n",
    "# Parse the html content\n",
    "\n",
    "soup = BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "# Get the title of the page\n",
    "title = soup.title.text\n",
    "\n",
    "# Get the content of the page\n",
    "content = soup.find('script', type = 'application/ld+json').string\n",
    "\n",
    "data = json.loads(content)\n",
    "\n",
    "article_body = data.get(\"@graph\", \"Text not found\")\n",
    "\n",
    "\n",
    "article_body = article_body[0].get(\"mainEntity\", \"Text not found\")\n",
    "\n",
    "\n",
    "# Get all the text from the article\n",
    "text = []\n",
    "\n",
    "print(article_body)\n",
    "\n",
    "for i in article_body:\n",
    "    text.append(i['acceptedAnswer']['text'])\n",
    "\n",
    "text =  ' '.join(text)\n",
    "\n",
    "# remove <p> tags\n",
    "text = text.replace('<p>', '')\n",
    "\n",
    "title, text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cnet add to db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "313115"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cnet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[33], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mlen\u001b[39m(cnet)):\n\u001b[1;32m      2\u001b[0m     url \u001b[38;5;241m=\u001b[39m cnet\u001b[38;5;241m.\u001b[39mloc[i, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mloc\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[0;32m----> 3\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mrequests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      4\u001b[0m     html \u001b[38;5;241m=\u001b[39m response\u001b[38;5;241m.\u001b[39mcontent\n\u001b[1;32m      5\u001b[0m     soup \u001b[38;5;241m=\u001b[39m BeautifulSoup(html, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhtml.parser\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/requests/api.py:73\u001b[0m, in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget\u001b[39m(url, params\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     63\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Sends a GET request.\u001b[39;00m\n\u001b[1;32m     64\u001b[0m \n\u001b[1;32m     65\u001b[0m \u001b[38;5;124;03m    :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     70\u001b[0m \u001b[38;5;124;03m    :rtype: requests.Response\u001b[39;00m\n\u001b[1;32m     71\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m---> 73\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mget\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/requests/api.py:59\u001b[0m, in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[1;32m     56\u001b[0m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[1;32m     57\u001b[0m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[1;32m     58\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m sessions\u001b[38;5;241m.\u001b[39mSession() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[0;32m---> 59\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/requests/sessions.py:589\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    584\u001b[0m send_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m    585\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimeout\u001b[39m\u001b[38;5;124m\"\u001b[39m: timeout,\n\u001b[1;32m    586\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_redirects\u001b[39m\u001b[38;5;124m\"\u001b[39m: allow_redirects,\n\u001b[1;32m    587\u001b[0m }\n\u001b[1;32m    588\u001b[0m send_kwargs\u001b[38;5;241m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 589\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    591\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/requests/sessions.py:747\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    744\u001b[0m         \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[1;32m    746\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m stream:\n\u001b[0;32m--> 747\u001b[0m     \u001b[43mr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcontent\u001b[49m\n\u001b[1;32m    749\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m r\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/requests/models.py:899\u001b[0m, in \u001b[0;36mResponse.content\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    897\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_content \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    898\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 899\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_content \u001b[38;5;241m=\u001b[39m \u001b[38;5;124;43mb\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43miter_content\u001b[49m\u001b[43m(\u001b[49m\u001b[43mCONTENT_CHUNK_SIZE\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    901\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_content_consumed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m    902\u001b[0m \u001b[38;5;66;03m# don't need to release the connection; that's been handled by urllib3\u001b[39;00m\n\u001b[1;32m    903\u001b[0m \u001b[38;5;66;03m# since we exhausted the data.\u001b[39;00m\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/requests/models.py:816\u001b[0m, in \u001b[0;36mResponse.iter_content.<locals>.generate\u001b[0;34m()\u001b[0m\n\u001b[1;32m    814\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mraw, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstream\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[1;32m    815\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 816\u001b[0m         \u001b[38;5;28;01myield from\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mraw\u001b[38;5;241m.\u001b[39mstream(chunk_size, decode_content\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m    817\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m ProtocolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    818\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m ChunkedEncodingError(e)\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/urllib3/response.py:934\u001b[0m, in \u001b[0;36mHTTPResponse.stream\u001b[0;34m(self, amt, decode_content)\u001b[0m\n\u001b[1;32m    932\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    933\u001b[0m     \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_fp_closed(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp) \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decoded_buffer) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m--> 934\u001b[0m         data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mamt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    936\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m data:\n\u001b[1;32m    937\u001b[0m             \u001b[38;5;28;01myield\u001b[39;00m data\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/urllib3/response.py:877\u001b[0m, in \u001b[0;36mHTTPResponse.read\u001b[0;34m(self, amt, decode_content, cache_content)\u001b[0m\n\u001b[1;32m    874\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decoded_buffer) \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m amt:\n\u001b[1;32m    875\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decoded_buffer\u001b[38;5;241m.\u001b[39mget(amt)\n\u001b[0;32m--> 877\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_raw_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    879\u001b[0m flush_decoder \u001b[38;5;241m=\u001b[39m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m (amt \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data)\n\u001b[1;32m    881\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decoded_buffer) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/urllib3/response.py:812\u001b[0m, in \u001b[0;36mHTTPResponse._raw_read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m    809\u001b[0m fp_closed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mclosed\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m    811\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_error_catcher():\n\u001b[0;32m--> 812\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fp_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m fp_closed \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    813\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m amt \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m data:\n\u001b[1;32m    814\u001b[0m         \u001b[38;5;66;03m# Platform-specific: Buggy versions of Python.\u001b[39;00m\n\u001b[1;32m    815\u001b[0m         \u001b[38;5;66;03m# Close the connection when no data is returned\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    820\u001b[0m         \u001b[38;5;66;03m# not properly close the connection in all cases. There is\u001b[39;00m\n\u001b[1;32m    821\u001b[0m         \u001b[38;5;66;03m# no harm in redundantly calling close.\u001b[39;00m\n\u001b[1;32m    822\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp\u001b[38;5;241m.\u001b[39mclose()\n",
      "File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/urllib3/response.py:797\u001b[0m, in \u001b[0;36mHTTPResponse._fp_read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m    794\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m buffer\u001b[38;5;241m.\u001b[39mgetvalue()\n\u001b[1;32m    795\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    796\u001b[0m     \u001b[38;5;66;03m# StringIO doesn't like amt=None\u001b[39;00m\n\u001b[0;32m--> 797\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp\u001b[38;5;241m.\u001b[39mread()\n",
      "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/http/client.py:459\u001b[0m, in \u001b[0;36mHTTPResponse.read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m    456\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    457\u001b[0m     \u001b[38;5;66;03m# Amount is given, implement using readinto\u001b[39;00m\n\u001b[1;32m    458\u001b[0m     b \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mbytearray\u001b[39m(amt)\n\u001b[0;32m--> 459\u001b[0m     n \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadinto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    460\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mmemoryview\u001b[39m(b)[:n]\u001b[38;5;241m.\u001b[39mtobytes()\n\u001b[1;32m    461\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    462\u001b[0m     \u001b[38;5;66;03m# Amount is not given (unbounded read) so we must check self.length\u001b[39;00m\n\u001b[1;32m    463\u001b[0m     \u001b[38;5;66;03m# and self.chunked\u001b[39;00m\n",
      "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/http/client.py:503\u001b[0m, in \u001b[0;36mHTTPResponse.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    498\u001b[0m         b \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmemoryview\u001b[39m(b)[\u001b[38;5;241m0\u001b[39m:\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlength]\n\u001b[1;32m    500\u001b[0m \u001b[38;5;66;03m# we do not use _safe_read() here because this may be a .will_close\u001b[39;00m\n\u001b[1;32m    501\u001b[0m \u001b[38;5;66;03m# connection, and the user is reading more bytes than will be provided\u001b[39;00m\n\u001b[1;32m    502\u001b[0m \u001b[38;5;66;03m# (for example, reading in 1k chunks)\u001b[39;00m\n\u001b[0;32m--> 503\u001b[0m n \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadinto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    504\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m n \u001b[38;5;129;01mand\u001b[39;00m b:\n\u001b[1;32m    505\u001b[0m     \u001b[38;5;66;03m# Ideally, we would raise IncompleteRead if the content-length\u001b[39;00m\n\u001b[1;32m    506\u001b[0m     \u001b[38;5;66;03m# wasn't satisfied, but it might break compatibility.\u001b[39;00m\n\u001b[1;32m    507\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_close_conn()\n",
      "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/socket.py:704\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    702\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m    703\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 704\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    705\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[1;32m    706\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/ssl.py:1241\u001b[0m, in \u001b[0;36mSSLSocket.recv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1237\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m flags \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m   1238\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1239\u001b[0m           \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m   1240\u001b[0m           \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m)\n\u001b[0;32m-> 1241\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbytes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1242\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1243\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mrecv_into(buffer, nbytes, flags)\n",
      "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/ssl.py:1099\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m   1097\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1098\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m buffer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1099\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1100\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1101\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sslobj\u001b[38;5;241m.\u001b[39mread(\u001b[38;5;28mlen\u001b[39m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for i in range(len(cnet)):\n",
    "    url = cnet.loc[i, 'loc']\n",
    "    response = requests.get(url)\n",
    "    html = response.content\n",
    "    soup = BeautifulSoup(html, 'html.parser')\n",
    "    title = soup.title.text\n",
    "    content = soup.find('script', type = 'application/ld+json')\n",
    "    if content is None:\n",
    "        continue\n",
    "    data = json.loads(content.string)\n",
    "    article_body = data.get(\"@graph\", \"Text not found\")\n",
    "    if article_body is None or len(article_body) < 900 or article_body == \"Text not found\":\n",
    "        continue\n",
    "    for i in article_body:\n",
    "        article_body = i.get(\"mainEntity\", \"Text not found\")\n",
    "    text = []\n",
    "    if article_body is None or len(article_body) < 900 or article_body == \"Text not found\" or type(article_bod) == str:\n",
    "        continue\n",
    "    for i in article_body:\n",
    "        text.append(i['acceptedAnswer']['text'])\n",
    "    text =  ' '.join(text)\n",
    "    if content is None or text is None or len(text) < 900:\n",
    "        continue\n",
    "    collection.add(\n",
    "        ids=url,\n",
    "        documents=text[:900],\n",
    "        metadatas={\n",
    "            \"title\": title,\n",
    "            \"url\": url,\n",
    "            \"source\": \"CNET\",\n",
    "        }\n",
    "    )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}