Skip to content

Commit

Permalink
update expired ebay parsing selector
Browse files Browse the repository at this point in the history
  • Loading branch information
mazen-r committed Aug 22, 2024
1 parent 5762858 commit 285040e
Show file tree
Hide file tree
Showing 5 changed files with 4,722 additions and 5,451 deletions.
15 changes: 6 additions & 9 deletions ebay-scraper/ebay.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,22 +116,19 @@ def parse_product(result: ScrapeApiResponse):
item["price_converted"] = css(".x-price-approx__price ::text") # ebay automatically converts price for some regions

item["name"] = css_join("h1 span::text")
item["seller_name"] = css_join("[data-testid=str-title] a ::text")
item["seller_url"] = css("[data-testid=str-title] a::attr(href)").split("?")[0]
item["seller_name"] = sel.xpath("//div[contains(@class,'info__about-seller')]/a/span/text()").get()
item["seller_url"] = sel.xpath("//div[contains(@class,'info__about-seller')]/a/@href").get().split("?")[0]
item["photos"] = sel.css('.ux-image-filmstrip-carousel-item.image img::attr("src")').getall() # carousel images
item["photos"].extend(sel.css('.ux-image-carousel-item.image img::attr("src")').getall()) # main image
# description is an iframe (independant page). We can keep it as an URL or scrape it later.
item["description_url"] = css("div.d-item-description iframe::attr(src)")
if not item["description_url"]:
item["description_url"] = css("div#desc_div iframe::attr(src)")
item["description_url"] = css("iframe#desc_ifr::attr(src)")
# feature details from the description table:
feature_table = sel.css("div.ux-layout-section--features")
features = {}
for ft_label in feature_table.css(".ux-labels-values__labels"):
for feature in feature_table.css("dl.ux-labels-values"):
# iterate through each label of the table and select first sibling for value:
label = "".join(ft_label.css(".ux-textspans::text").getall()).strip(":\n ")
ft_value = ft_label.xpath("following-sibling::div[1]")
value = "".join(ft_value.css(".ux-textspans::text").getall()).strip()
label = "".join(feature.css(".ux-labels-values__labels-content > div > span::text").getall()).strip(":\n ")
value = "".join(feature.css(".ux-labels-values__values-content > div > span *::text").getall()).strip(":\n ")
features[label] = value
item["features"] = features
return item
Expand Down
60 changes: 28 additions & 32 deletions ebay-scraper/results/product-with-variants.json
Original file line number Diff line number Diff line change
@@ -1,22 +1,18 @@
{
"url": "https://www.ebay.com/itm/393531906094",
"id": "393531906094",
"price_original": "C $469.00",
"price_converted": "US $341.55",
"price_original": "C $439.99",
"price_converted": "US $322.82",
"name": "Apple iPhone 11 Pro Max - Unlocked - 64GB / 256GB / 512GB - CA - Excellent",
"seller_name": "MobileKlinik",
"seller_url": "https://www.ebay.com/str/devicecare",
"photos": [
"https://i.ebayimg.com/images/g/93cAAOSwvEJgbLW8/s-l64.jpg",
"https://i.ebayimg.com/images/g/l0sAAOSwextgbLYj/s-l64.jpg",
"https://i.ebayimg.com/images/g/qxEAAOSwP~BgbLa5/s-l64.jpg",
"https://i.ebayimg.com/images/g/7usAAOSwRbZgbLbE/s-l64.jpg",
"https://i.ebayimg.com/images/g/ffMAAOSwhAxgbLbO/s-l64.jpg",
"https://i.ebayimg.com/images/g/93cAAOSwvEJgbLW8/s-l500.jpg"
"https://i.ebayimg.com/thumbs/images/g/93cAAOSwvEJgbLW8/s-l500.jpg",
"https://i.ebayimg.com/images/g/93cAAOSwvEJgbLW8/s-l1600.webp"
],
"description_url": "https://vi.vipr.ebaydesc.com/ws/eBayISAPI.dll?ViewItemDescV4&item=393531906094&t=1631237959000&category=9355&seller=mobileklinik&excSoj=1&excTrk=1&lsite=2&ittenable=true&domain=ebay.com&descgauge=1&cspheader=1&oneClk=2&secureDesc=1",
"description_url": "https://vi.vipr.ebaydesc.com/itmdesc/393531906094?t=1631237959000&category=9355&seller=mobileklinik&excSoj=1&ver=0&excTrk=1&lsite=2&ittenable=true&domain=ebay.com&descgauge=1&cspheader=1&oneClk=2&secureDesc=1&variationId=662315637172",
"features": {
"Condition": "Excellent - Refurbished: The item is in like-new condition, backed by a one year warranty. It has ... Read moreExcellent - Refurbished: The item is in like-new condition, backed by a one year warranty. It has been professionally refurbished, inspected and cleaned to excellent condition by qualified sellers. The item includes original or new accessories and will come in new generic packaging. See the seller's listing for full details. See all condition definitions",
"Condition": "Excellent - Refurbished: The item is in like-new condition, backed by a one year warranty. It has ... Read moreabout the conditionExcellent - Refurbished: The item is in like-new condition, backed by a one year warranty. It has been professionally refurbished, inspected and cleaned to excellent condition by qualified sellers. The item includes original or new accessories and will come in new generic packaging. See the seller's listing for full details. See all condition definitionsopens in a new window or tab",
"Processor": "Hexa Core",
"Screen Size": "6.5 in",
"Model Number": "A2161 (CDMA + GSM)",
Expand All @@ -39,7 +35,7 @@
"Color": "Midnight Green",
"price_original": 679,
"price_original_currency": "CAD",
"price_converted": 494.48,
"price_converted": 498.18,
"price_converted_currency": "USD",
"out_of_stock": true
},
Expand All @@ -50,7 +46,7 @@
"Color": "Gold",
"price_original": 679,
"price_original_currency": "CAD",
"price_converted": 494.48,
"price_converted": 498.18,
"price_converted_currency": "USD",
"out_of_stock": true
},
Expand All @@ -61,7 +57,7 @@
"Color": "Space Gray",
"price_original": 679,
"price_original_currency": "CAD",
"price_converted": 494.48,
"price_converted": 498.18,
"price_converted_currency": "USD",
"out_of_stock": true
},
Expand All @@ -70,9 +66,9 @@
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "256 GB",
"Color": "Midnight Green",
"price_original": 549,
"price_original": 489,
"price_original_currency": "CAD",
"price_converted": 399.81,
"price_converted": 358.78,
"price_converted_currency": "USD",
"out_of_stock": true
},
Expand All @@ -81,9 +77,9 @@
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "256 GB",
"Color": "Gold",
"price_original": 549,
"price_original": 489,
"price_original_currency": "CAD",
"price_converted": 399.81,
"price_converted": 358.78,
"price_converted_currency": "USD",
"out_of_stock": true
},
Expand All @@ -92,11 +88,11 @@
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "256 GB",
"Color": "Space Gray",
"price_original": 549,
"price_original": 489,
"price_original_currency": "CAD",
"price_converted": 399.81,
"price_converted": 358.78,
"price_converted_currency": "USD",
"out_of_stock": true
"out_of_stock": false
},
{
"id": "662315637179",
Expand All @@ -105,7 +101,7 @@
"Color": "Silver",
"price_original": 679,
"price_original_currency": "CAD",
"price_converted": 494.48,
"price_converted": 498.18,
"price_converted_currency": "USD",
"out_of_stock": true
},
Expand All @@ -114,20 +110,20 @@
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "64 GB",
"Color": "Midnight Green",
"price_original": 469,
"price_original": 439.99,
"price_original_currency": "CAD",
"price_converted": 341.55,
"price_converted": 322.82,
"price_converted_currency": "USD",
"out_of_stock": true
"out_of_stock": false
},
{
"id": "662315637173",
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "64 GB",
"Color": "Gold",
"price_original": 469,
"price_original": 439.99,
"price_original_currency": "CAD",
"price_converted": 341.55,
"price_converted": 322.82,
"price_converted_currency": "USD",
"out_of_stock": false
},
Expand All @@ -136,9 +132,9 @@
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "64 GB",
"Color": "Space Gray",
"price_original": 469,
"price_original": 439.99,
"price_original_currency": "CAD",
"price_converted": 341.55,
"price_converted": 322.82,
"price_converted_currency": "USD",
"out_of_stock": false
},
Expand All @@ -147,9 +143,9 @@
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "256 GB",
"Color": "Silver",
"price_original": 549,
"price_original": 489,
"price_original_currency": "CAD",
"price_converted": 399.81,
"price_converted": 358.78,
"price_converted_currency": "USD",
"out_of_stock": true
},
Expand All @@ -158,9 +154,9 @@
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "64 GB",
"Color": "Silver",
"price_original": 469,
"price_original": 439.99,
"price_original_currency": "CAD",
"price_converted": 341.55,
"price_converted": 322.82,
"price_converted_currency": "USD",
"out_of_stock": true
}
Expand Down
176 changes: 158 additions & 18 deletions ebay-scraper/results/product.json
Original file line number Diff line number Diff line change
@@ -1,24 +1,164 @@
{
"url": "https://www.ebay.com/itm/332562282948",
"id": "332562282948",
"price_original": "US $12.45",
"price_converted": "",
"name": "Sanei Kirby 5.5\" Plush Stuffed Doll (KP01) - Kirby Adventure All Star Collection",
"seller_name": "ToysCollections",
"seller_url": "https://www.ebay.com/str/huskylover228",
"url": "https://www.ebay.com/itm/393531906094",
"id": "393531906094",
"price_original": "C $439.99",
"price_converted": "US $322.82",
"name": "Apple iPhone 11 Pro Max - Unlocked - 64GB / 256GB / 512GB - CA - Excellent",
"seller_name": "MobileKlinik",
"seller_url": "https://www.ebay.com/str/devicecare",
"photos": [
"https://i.ebayimg.com/images/g/ITEAAOSw9p9ajK16/s-l500.jpg"
"https://i.ebayimg.com/thumbs/images/g/93cAAOSwvEJgbLW8/s-l500.jpg",
"https://i.ebayimg.com/images/g/93cAAOSwvEJgbLW8/s-l1600.webp"
],
"description_url": "https://vi.vipr.ebaydesc.com/ws/eBayISAPI.dll?ViewItemDescV4&item=332562282948&t=1678153940000&category=69528&seller=the_northeshop&excSoj=1&excTrk=1&lsite=0&ittenable=true&domain=ebay.com&descgauge=1&cspheader=1&oneClk=2&secureDesc=1",
"description_url": "https://vi.vipr.ebaydesc.com/itmdesc/393531906094?t=1631237959000&category=9355&seller=mobileklinik&excSoj=1&ver=0&excTrk=1&lsite=2&ittenable=true&domain=ebay.com&descgauge=1&cspheader=1&oneClk=2&secureDesc=1&variationId=662315637172",
"features": {
"Condition": "New: A brand-new, unused, unopened, undamaged item (including handmade items). See the seller's ... Read moreNew: A brand-new, unused, unopened, undamaged item (including handmade items). See the seller's listing for full details. See all condition definitions",
"Brand": "unbranded",
"Type": "Plush",
"UPC": "4905330122810",
"Featured Refinements": "Kirby Plush",
"Recommended Age Range": "4+",
"Gender": "Boys & Girls",
"Character Family": "Kirby Adventure"
"Condition": "Excellent - Refurbished: The item is in like-new condition, backed by a one year warranty. It has ... Read moreabout the conditionExcellent - Refurbished: The item is in like-new condition, backed by a one year warranty. It has been professionally refurbished, inspected and cleaned to excellent condition by qualified sellers. The item includes original or new accessories and will come in new generic packaging. See the seller's listing for full details. See all condition definitionsopens in a new window or tab",
"Processor": "Hexa Core",
"Screen Size": "6.5 in",
"Model Number": "A2161 (CDMA + GSM)",
"Lock Status": "Factory Unlocked",
"SIM Card Slot": "Dual SIM (SIM + eSIM)",
"Brand": "Apple",
"Network": "1&1, Unlocked",
"Connectivity": "5G, Bluetooth, GPS, Lightning",
"Operating System": "iOS",
"Features": "4K Video Recording, Accelerometer, Bluetooth Enabled, Camera, Facial Recognition",
"Contract": "Without Contract",
"Camera Resolution": "12.0 MP",
"RAM": "4 GB"
},
"variants": {}
"variants": [
{
"id": "662315637180",
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "512 GB",
"Color": "Midnight Green",
"price_original": 679,
"price_original_currency": "CAD",
"price_converted": 498.18,
"price_converted_currency": "USD",
"out_of_stock": true
},
{
"id": "662315637181",
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "512 GB",
"Color": "Gold",
"price_original": 679,
"price_original_currency": "CAD",
"price_converted": 498.18,
"price_converted_currency": "USD",
"out_of_stock": true
},
{
"id": "662315637182",
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "512 GB",
"Color": "Space Gray",
"price_original": 679,
"price_original_currency": "CAD",
"price_converted": 498.18,
"price_converted_currency": "USD",
"out_of_stock": true
},
{
"id": "662315637176",
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "256 GB",
"Color": "Midnight Green",
"price_original": 489,
"price_original_currency": "CAD",
"price_converted": 358.78,
"price_converted_currency": "USD",
"out_of_stock": true
},
{
"id": "662315637177",
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "256 GB",
"Color": "Gold",
"price_original": 489,
"price_original_currency": "CAD",
"price_converted": 358.78,
"price_converted_currency": "USD",
"out_of_stock": true
},
{
"id": "662315637178",
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "256 GB",
"Color": "Space Gray",
"price_original": 489,
"price_original_currency": "CAD",
"price_converted": 358.78,
"price_converted_currency": "USD",
"out_of_stock": false
},
{
"id": "662315637179",
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "512 GB",
"Color": "Silver",
"price_original": 679,
"price_original_currency": "CAD",
"price_converted": 498.18,
"price_converted_currency": "USD",
"out_of_stock": true
},
{
"id": "662315637172",
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "64 GB",
"Color": "Midnight Green",
"price_original": 439.99,
"price_original_currency": "CAD",
"price_converted": 322.82,
"price_converted_currency": "USD",
"out_of_stock": false
},
{
"id": "662315637173",
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "64 GB",
"Color": "Gold",
"price_original": 439.99,
"price_original_currency": "CAD",
"price_converted": 322.82,
"price_converted_currency": "USD",
"out_of_stock": false
},
{
"id": "662315637174",
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "64 GB",
"Color": "Space Gray",
"price_original": 439.99,
"price_original_currency": "CAD",
"price_converted": 322.82,
"price_converted_currency": "USD",
"out_of_stock": false
},
{
"id": "662315637175",
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "256 GB",
"Color": "Silver",
"price_original": 489,
"price_original_currency": "CAD",
"price_converted": 358.78,
"price_converted_currency": "USD",
"out_of_stock": true
},
{
"id": "662315637171",
"Model": "Apple iPhone 11 Pro Max",
"Storage Capacity": "64 GB",
"Color": "Silver",
"price_original": 439.99,
"price_original_currency": "CAD",
"price_converted": 322.82,
"price_converted_currency": "USD",
"out_of_stock": true
}
]
}
Loading

0 comments on commit 285040e

Please sign in to comment.