From c2754edebc96ac7cf86fba80cce2f182dba20bca Mon Sep 17 00:00:00 2001 From: "J. Hathaway" Date: Tue, 14 Apr 2026 15:26:40 -0600 Subject: [PATCH 01/34] Refactor introduction and update contributor list Removed duplicate sentences and improved clarity of the introduction. Updated contributor list formatting. --- index.md | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/index.md b/index.md index 965c550..6086806 100644 --- a/index.md +++ b/index.md @@ -5,29 +5,14 @@ aliases: # Welcome -[![DOI](https://zenodo.org/badge/496994611.svg)](https://zenodo.org/doi/10.5281/zenodo.10518241) ![GitHub Release](https://img.shields.io/github/v/release/aeturrell/python4DS) +This is the website for **Python for Data Science**, a book heavily inspired by the excellent [**R for Data Science (2e)**](https://r4ds.hadley.nz/). This book will teach you how to load up, transform, visualize, and begin to understand your data. The book aims to give you the skills you need to code for data science. It's suitable for people who have some familiarity with the ideas behind programming and coding but who don't yet know how to do data science. -This is the website for **Python for Data Science**, a book heavily inspired by the excellent [**R for Data Science (2e)**](https://r4ds.hadley.nz/). This book will teach you how to load up, transform, visualise, and begin to understand your data. The book aims to give you the skills you need to code for data science. It's suitable for people who have some familiarity with the ideas behind programming and coding but who don't yet know how to do data science. +This book teaches you how to do data science using one of the world's most popular programming languages, Python. While Python is a general-purpose language, meaning it is used for a wide range of tasks, it is also the most widely used language for data science (though note that both SQL and R are also used for data science). -This book teaches you how to do data science using one of the world's most popular programming languages, Python. While Python is a general purpose language, which means it is used for a wide range of tasks, it is also the most widely used language for data science (although note that both SQL and R are also used for data science). +__This [fork](https://github.com/aeturrell/python4DS) of a [fork](https://github.com/hadley/r4ds) focuses on Polars instead of Pandas for data wrangling.__ To begin your data science journey, head to the next page. ## Contributors to Python4DS Contributing is very much encouraged. If you're looking for content to implement or tweak, we aim to follow the structure and content of **R for Data Science (2e)** and you can find open [issues here](https://github.com/aeturrell/python4DS/issues). For larger contributions of content, it's probably best to check with other contributors first. - -We thank the following contributors: - -- [Arthur Turrell](https://aeturrell.com/), who has also contributed to [*Coding for Economists*](https://aeturrell.github.io/coding-for-economists) and wrote popular non-fiction book [*The Star Builders*](https://aeturrell.com/thestarbuilders/thestarbuilders.html) -- [Pietro Monticone](https://github.com/pitmonticone) -- [Antonio Mele](https://github.com/meleantonio) -- [Igor Alshannikov](https://github.com/alshan) -- [Umair Durrani](https://github.com/durraniu) -- [Zeki Akyol](https://github.com/zekiakyol) -- [Yiben Huang](https://github.com/yibenhuang) -- [William Chiu](https://github.com/crossxwill) -- [udurraniAtPresage](https://github.com/udurraniAtPresage) -- [Josh Holman](https://github.com/TheJolman) -- [Kenytt Avery](https://github.com/ProfAvery) -- [Bradley Phipps](https://github.com/hotshotberad) From f1ddaa171bcf8b6d3a8fa428530927a6602e9f8c Mon Sep 17 00:00:00 2001 From: "J. Hathaway" Date: Tue, 14 Apr 2026 15:31:58 -0600 Subject: [PATCH 02/34] Create build_book.yaml --- .github/workflows/build_book.yaml | 70 +++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/build_book.yaml diff --git a/.github/workflows/build_book.yaml b/.github/workflows/build_book.yaml new file mode 100644 index 0000000..0842bb8 --- /dev/null +++ b/.github/workflows/build_book.yaml @@ -0,0 +1,70 @@ +name: Build and Deploy Quarto Book + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + schedule: + - cron: "0 23 * * *" # run every day at 11 PM UTC + +env: + RUST_BACKTRACE: 1 + +permissions: + contents: read + pages: write + id-token: write + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install Python dependencies + if: hashFiles('requirements.txt') != '' + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Install Jupyter basics (if not in requirements) + if: hashFiles('requirements.txt') == '' + run: | + python -m pip install --upgrade pip + pip install jupyter nbconvert ipykernel + + - name: Set up Quarto + uses: quarto-dev/quarto-actions/setup@v2 + with: + tinytex: true + + - name: Render Quarto book + run: | + quarto render + + - name: Upload website artifact + if: ${{ github.ref == 'refs/heads/main' }} + uses: actions/upload-pages-artifact@v3 + with: + path: _book + + deploy: + if: ${{ github.ref == 'refs/heads/main' }} + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 From 4b78eaf96818c9772b00531f35e66e88d32ceec9 Mon Sep 17 00:00:00 2001 From: "J. Hathaway" Date: Tue, 14 Apr 2026 15:32:53 -0600 Subject: [PATCH 03/34] Add GitHub Actions workflow for Quarto book build --- .github/workflows/build_book2.yaml | 70 ++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/build_book2.yaml diff --git a/.github/workflows/build_book2.yaml b/.github/workflows/build_book2.yaml new file mode 100644 index 0000000..0842bb8 --- /dev/null +++ b/.github/workflows/build_book2.yaml @@ -0,0 +1,70 @@ +name: Build and Deploy Quarto Book + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + schedule: + - cron: "0 23 * * *" # run every day at 11 PM UTC + +env: + RUST_BACKTRACE: 1 + +permissions: + contents: read + pages: write + id-token: write + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install Python dependencies + if: hashFiles('requirements.txt') != '' + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Install Jupyter basics (if not in requirements) + if: hashFiles('requirements.txt') == '' + run: | + python -m pip install --upgrade pip + pip install jupyter nbconvert ipykernel + + - name: Set up Quarto + uses: quarto-dev/quarto-actions/setup@v2 + with: + tinytex: true + + - name: Render Quarto book + run: | + quarto render + + - name: Upload website artifact + if: ${{ github.ref == 'refs/heads/main' }} + uses: actions/upload-pages-artifact@v3 + with: + path: _book + + deploy: + if: ${{ github.ref == 'refs/heads/main' }} + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 From 7c8012cec9b9a6128f6d9e7db9ee0f85a62e20a3 Mon Sep 17 00:00:00 2001 From: "J. Hathaway" Date: Tue, 14 Apr 2026 15:33:38 -0600 Subject: [PATCH 04/34] Delete .github/workflows/build_book.yaml --- .github/workflows/build_book.yaml | 70 ------------------------------- 1 file changed, 70 deletions(-) delete mode 100644 .github/workflows/build_book.yaml diff --git a/.github/workflows/build_book.yaml b/.github/workflows/build_book.yaml deleted file mode 100644 index 0842bb8..0000000 --- a/.github/workflows/build_book.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: Build and Deploy Quarto Book - -on: - push: - branches: [main] - pull_request: - branches: [main] - workflow_dispatch: - schedule: - - cron: "0 23 * * *" # run every day at 11 PM UTC - -env: - RUST_BACKTRACE: 1 - -permissions: - contents: read - pages: write - id-token: write - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Python dependencies - if: hashFiles('requirements.txt') != '' - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: Install Jupyter basics (if not in requirements) - if: hashFiles('requirements.txt') == '' - run: | - python -m pip install --upgrade pip - pip install jupyter nbconvert ipykernel - - - name: Set up Quarto - uses: quarto-dev/quarto-actions/setup@v2 - with: - tinytex: true - - - name: Render Quarto book - run: | - quarto render - - - name: Upload website artifact - if: ${{ github.ref == 'refs/heads/main' }} - uses: actions/upload-pages-artifact@v3 - with: - path: _book - - deploy: - if: ${{ github.ref == 'refs/heads/main' }} - needs: build - runs-on: ubuntu-latest - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 From 17052a54cc42626ae89dfb6f454d67e3676d1a84 Mon Sep 17 00:00:00 2001 From: "J. Hathaway" Date: Tue, 14 Apr 2026 15:41:17 -0600 Subject: [PATCH 05/34] Update GitHub Actions workflow for book build --- .github/workflows/build_book2.yaml | 47 ++++++++++++++++-------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build_book2.yaml b/.github/workflows/build_book2.yaml index 0842bb8..563a836 100644 --- a/.github/workflows/build_book2.yaml +++ b/.github/workflows/build_book2.yaml @@ -7,50 +7,53 @@ on: branches: [main] workflow_dispatch: schedule: - - cron: "0 23 * * *" # run every day at 11 PM UTC - -env: - RUST_BACKTRACE: 1 + - cron: "0 23 * * *" # daily at 23:00 UTC permissions: contents: read pages: write id-token: write +concurrency: + group: "pages" + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest + steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" + - name: Setup Pages + uses: actions/configure-pages@v5 - - name: Install Python dependencies - if: hashFiles('requirements.txt') != '' - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + # Use the Python version your project expects + python-version: "3.12" - - name: Install Jupyter basics (if not in requirements) - if: hashFiles('requirements.txt') == '' + - name: Sync project environment run: | - python -m pip install --upgrade pip - pip install jupyter nbconvert ipykernel + uv sync - - name: Set up Quarto + - name: Install Quarto uses: quarto-dev/quarto-actions/setup@v2 with: - tinytex: true + tinytex: false + + - name: Render book with uv + run: | + uv run quarto render --execute - - name: Render Quarto book + - name: Copy data folder into book output run: | - quarto render + mkdir -p _book/data + cp -r data/* _book/data/ - - name: Upload website artifact + - name: Upload site artifact if: ${{ github.ref == 'refs/heads/main' }} uses: actions/upload-pages-artifact@v3 with: From 363f4201f10e7a4e12bd94b4f575031d94ca87f4 Mon Sep 17 00:00:00 2001 From: "J. Hathaway" Date: Tue, 14 Apr 2026 15:50:26 -0600 Subject: [PATCH 06/34] Delete .github/workflows/build_book2.yaml --- .github/workflows/build_book2.yaml | 73 ------------------------------ 1 file changed, 73 deletions(-) delete mode 100644 .github/workflows/build_book2.yaml diff --git a/.github/workflows/build_book2.yaml b/.github/workflows/build_book2.yaml deleted file mode 100644 index 563a836..0000000 --- a/.github/workflows/build_book2.yaml +++ /dev/null @@ -1,73 +0,0 @@ -name: Build and Deploy Quarto Book - -on: - push: - branches: [main] - pull_request: - branches: [main] - workflow_dispatch: - schedule: - - cron: "0 23 * * *" # daily at 23:00 UTC - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Pages - uses: actions/configure-pages@v5 - - - name: Install uv - uses: astral-sh/setup-uv@v3 - with: - # Use the Python version your project expects - python-version: "3.12" - - - name: Sync project environment - run: | - uv sync - - - name: Install Quarto - uses: quarto-dev/quarto-actions/setup@v2 - with: - tinytex: false - - - name: Render book with uv - run: | - uv run quarto render --execute - - - name: Copy data folder into book output - run: | - mkdir -p _book/data - cp -r data/* _book/data/ - - - name: Upload site artifact - if: ${{ github.ref == 'refs/heads/main' }} - uses: actions/upload-pages-artifact@v3 - with: - path: _book - - deploy: - if: ${{ github.ref == 'refs/heads/main' }} - needs: build - runs-on: ubuntu-latest - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 From f8097a91531ee2783499ce34657774bdb8db7807 Mon Sep 17 00:00:00 2001 From: "J. Hathaway" Date: Tue, 14 Apr 2026 16:01:28 -0600 Subject: [PATCH 07/34] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 016e885..14ed0d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "python4ds" -version = "1.0.5" +version = "0.0.1" description = "The online book that teaches you how to use Python for data science." readme = "README.md" requires-python = ">=3.12.0,<3.13" From 3fa79101683fe8df1bb7ccbfbf27345101bb8520 Mon Sep 17 00:00:00 2001 From: Nwabueze Ugoh Date: Fri, 17 Apr 2026 13:19:13 +0100 Subject: [PATCH 08/34] Update spreadsheet.ipynb and databases.ipynb to replace pandas with polars. --- databases.ipynb | 36 +- pyproject.toml | 2 + spreadsheets.ipynb | 907 +++++++++++++++++++++++---------------------- uv.lock | 28 +- 4 files changed, 511 insertions(+), 462 deletions(-) diff --git a/databases.ipynb b/databases.ipynb index f14914c..b295748 100644 --- a/databases.ipynb +++ b/databases.ipynb @@ -17,7 +17,7 @@ "\n", "### Prerequisites\n", "\n", - "You will need the **pandas**, **SQLModel**, and **ibis** packages for this chapter. You probably already have **pandas** installed; to install **SQLModel** and **ibis** respectively run `uv add sqlmodel` and `uv add ibis-framework` on your computer's command line. First, let's bring in some general packages and turn off verbose warnings." + "You will need the **polars**, **SQLModel**, and **ibis** packages for this chapter. You probably already have **polars** installed; to install **SQLModel** and **ibis** respectively run `uv add sqlmodel` and `uv add ibis-framework` on your computer's command line. First, let's bring in some general packages and turn off verbose warnings." ] }, { @@ -39,10 +39,9 @@ "metadata": {}, "source": [ "## Database Basics\n", - "\n", - "At the simplest level, you can think about a database as a collection of data frames, called **tables** in database terminology.\n", - "Like a **pandas** data frame, a database table is a collection of named columns, where every value in the column is the same type.\n", - "There are three high level differences between data frames and database tables:\n", + "At the simplest level, you can think about a database as a collection of data frames, called **tables** in database terminology. \n", + "Like a **Polars** DataFrame, a database table is a collection of named columns, where every value in a column shares the same data type. \n", + "There are three high-level differences between data frames and database tables:\n", "\n", "- Database tables are stored on disk (ie on file) and can be arbitrarily large.\n", " Data frames are stored in memory, and are fundamentally limited (although that limit is still big enough for many problems). You can think about the difference between on disk and in memory as being like the difference between long-term and short-term memory (and you have much more limited capacity in the latter).\n", @@ -68,7 +67,7 @@ "\n", "- You'll always use a database interface that provides a connection to the database, for example Python's built-in **sqlite** package\n", "\n", - "- You'll also use a package that pushes and/or pulls data to/from the database, for example **pandas**\n", + "- You'll also use a package that pushes and/or pulls data to/from the database, for example **polars**\n", "\n", "The precise details of the connection varies a lot from DBMS to DBMS so unfortunately we can't cover all the details here. The initial setup will often take a little fiddling (and maybe some research) to get right, but you'll generally only need to do it once. We'll do the best we can to cover some basics here.\n", "\n", @@ -112,7 +111,7 @@ "id": "2992b718", "metadata": {}, "source": [ - "Note that the output here is in the form a Python object called a tuple. If we wanted to put this into a **pandas** data frame, we can just pass it straight in:" + "Note that the output here is in the form of a Python object called a tuple. If we want to convert this into a **Polars** DataFrame, we can pass it to `pl.DataFrame()`. When working with tuples, you may need to provide column names using the **schema** argument or specify **orient=\"row\"** so Polars correctly interprets the structure." ] }, { @@ -122,9 +121,11 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import polars as pl\n", + "\n", + "df = pl.DataFrame(rows, orient=\"row\")\n", "\n", - "pd.DataFrame(rows)" + "df" ] }, { @@ -316,9 +317,9 @@ "source": [ "### Joins\n", "\n", - "If you're familiar with joins in **pandas**, SQL joins are very similar. Let's see if we can join the 'album' and 'track' tables to find the *name* of the albums in the above query.\n", + "If you’re familiar with joins in **polars**, SQL joins are very similar. Let’s see if we can join the 'album' and 'track' tables to find the *name* of the albums in the above query.\n", "\n", - "Note that as soon as we have the *same* column names in more than one table, we need to specify the table we are referring to when we use that column name. There are different options for joins (eg `INNER`, `LEFT`) that you can find out more about [here](https://en.wikipedia.org/wiki/Join_(SQL)).\n" + "In polars, you use the `df.join()` method, which defaults to an \"inner\" join. Note that if you have the same column names in both tables, Polars will often append a suffix (like _right) to the duplicate names to keep them distinct, unless you specify otherwise. There are different options for joins (eg `INNER`, `LEFT`) that you can find out more about [here](https://en.wikipedia.org/wiki/Join_(SQL)).\n" ] }, { @@ -403,9 +404,9 @@ "id": "495f97e5", "metadata": {}, "source": [ - "## SQL with **pandas**\n", + "## SQL with **polars**\n", "\n", - "**pandas** is well-equipped for working with SQL. We can simply push the query we just created straight through using its `read_sql()` function—but bear in mind we need to pass in the connection we created to the database too:" + "**polars** is well-equipped for working with SQL. We can simply push the query we just created straight through using its `read_database()` function—but bear in mind we need to pass in the connection we created to the database too:" ] }, { @@ -415,7 +416,10 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_sql(sql_join, con)" + "df = pl.read_database(\n", + " query=sql_join, # your SQL query (string)\n", + " connection=con # your connection object (SQLAlchemy, psycopg2 cursor, etc.)\n", + ")" ] }, { @@ -435,7 +439,7 @@ "source": [ "## SQL with **ibis**\n", "\n", - "It's not exactly satisfactory to have to write out your SQL queries in text. What if we could create commands directly from **pandas** commands? You can't *quite* do that, but there's a package that gets you pretty close and it's called [**ibis**](https://ibis-project.org/). **ibis** is particularly useful when you are reading from a database and want to query it just like you would a **pandas** data frame.\n", + "It's not exactly satisfactory to have to write out your SQL queries in text. What if we could create commands directly from **polars** commands? You can't *quite* do that, but there's a package that gets you pretty close and it's called [**ibis**](https://ibis-project.org/). **ibis** is particularly useful when you are reading from a database and want to query it just like you would a **polars** data frame.\n", "\n", "**Ibis** can connect to local databases (eg a SQLite database), server-based databases (eg Postgres), or cloud-based databased (eg Google's BigQuery). The syntax to make a connection is, for example, `ibis.bigquery.connect`.\n", "\n", @@ -462,7 +466,7 @@ "id": "6dcd7d71", "metadata": {}, "source": [ - "Okay, now let's reproduce the following query: \"SELECT albumid, AVG(milliseconds)/1e3/60 FROM track GROUP BY albumid ORDER BY AVG(milliseconds) ASC LIMIT 5;\". We'll use a groupby, a mutate (which you can think of like **pandas**' assign statement), a sort, and then `limit()` to only show the first five entries." + "Okay, now let's reproduce the following query: \"SELECT albumid, AVG(milliseconds)/1e3/60 FROM track GROUP BY albumid ORDER BY AVG(milliseconds) ASC LIMIT 5;\". We'll use a group_by, a mutate (which you can think of like **polars** assign statement), a sort, and then `limit()` to only show the first five entries." ] }, { diff --git a/pyproject.toml b/pyproject.toml index 14ed0d8..7740186 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ readme = "README.md" requires-python = ">=3.12.0,<3.13" dependencies = [ "beautifulsoup4>=4.12.3", + "fastexcel>=0.19.0", "graphviz>=0.20.3", "ibis-framework[sqlite]>=9.5.0", "ipykernel>=6.29.5", @@ -36,6 +37,7 @@ dependencies = [ "toml>=0.10.2", "watermark>=2.5.0", "wbgapi>=1.0.14", + "xlsxwriter>=3.2.0", "yfinance>=1.2.1", ] diff --git a/spreadsheets.ipynb b/spreadsheets.ipynb index 12f1672..8053af7 100644 --- a/spreadsheets.ipynb +++ b/spreadsheets.ipynb @@ -1,447 +1,464 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "95f0a171", - "metadata": {}, - "source": [ - "# Spreadsheets {#sec-spreadsheets}\n", - "\n", - "## Introduction\n", - "\n", - "This chapter will show you how to work with spreadsheets, for example Microsoft Excel files, in Python. We already saw how to import csv (and tsv) files in @sec-data-import. In this chapter we will introduce you to tools for working with data in Excel spreadsheets and Google Sheets.\n", - "\n", - "If you or your collaborators are using spreadsheets for organising data that will be ingested by an analytical tool like Python, we recommend reading the paper \"Data Organization in Spreadsheets\" by Karl Broman and Kara Woo {cite}`broman2018data`. The best practices presented in this paper will save you much headache down the line when you import the data from a spreadsheet into Python to analyse and visualise. (For spreadsheets that are meant to be read by humans, we recommend the [good practice tables](https://github.com/best-practice-and-impact/gptables) package.)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "51a55374", - "metadata": { - "tags": [ - "remove-cell" - ] - }, - "outputs": [], - "source": [ - "# remove cell\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib_inline.backend_inline\n", - "\n", - "# Plot settings\n", - "plt.style.use(\"https://github.com/aeturrell/python4DS/raw/main/plot_style.txt\")\n", - "matplotlib_inline.backend_inline.set_matplotlib_formats(\"svg\")" - ] - }, - { - "cell_type": "markdown", - "id": "17575f3a", - "metadata": {}, - "source": [ - "### Prerequisites\n", - "\n", - "You will need to install the **pandas** package for this chapter. You will also need to install the **openpyxl** package by running `uv add openpyxl` in the terminal." - ] - }, - { - "cell_type": "markdown", - "id": "ff99055a", - "metadata": {}, - "source": [ - "## Reading Excel (and Similar) Files\n", - "\n", - "**pandas** can read in xls, xlsx, xlsm, xlsb, odf, ods, and odt files from your local filesystem or from a URL. It also supports an option to read a single sheet or a list of sheets.\n", - "\n", - "To show how this works, we'll work with an example spreadsheet called \"students.xlsx\". The figure below shows what the spreadsheet looks like.\n", - "\n", - "![A look at the students spreadsheet in Excel. The spreadsheet contains information on 6 students, their ID, full name, favourite food, meal plan, and age.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-students.png)" - ] - }, - { - "cell_type": "markdown", - "id": "29f2f4e0", - "metadata": {}, - "source": [ - "The first argument to `pd.read_excel()` is the path to the file to read. If you have downloaded the [file]() onto your computer and put it in a subfolder called \"data\" then you would want to use the path \"data/students.xlsx\" but we can also load it directly from the URL." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bf00d4c6", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "students = pd.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/students.xlsx\"\n", - ")\n", - "students" - ] - }, - { - "cell_type": "markdown", - "id": "03d6ba9c", - "metadata": {}, - "source": [ - "We have six students in the data and five variables on each student. However there are a few things we might want to address in this dataset:\n", - "\n", - "- The column names are all over the place. You can provide column names that follow a consistent format; we recommend `snake_case` using the `names` argument.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ecce26ea", - "metadata": {}, - "outputs": [], - "source": [ - "pd.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/students.xlsx\",\n", - " names=[\"student_id\", \"full_name\", \"favourite_food\", \"meal_plan\", \"age\"],\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "bb07ad4f", - "metadata": {}, - "source": [ - "\n", - "- `age` is read in as a column of objects, but it really should be numeric. Just like with `read_csv()`, you can supply a `dtype` argument to `read_excel()` and specify the data types for the columns of data you read in. Your options include `\"boolean\"`, `\"int\"`, `\"float\"`, `\"datetime\"`, `\"string\"`, and more. But we can see right away that this isn't going to work with the \"age\" column as it mixes numbers and text: so we first need to map its text to numbers." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "22e45cc2", - "metadata": {}, - "outputs": [], - "source": [ - "students = pd.read_excel(\n", - " \"data/students.xlsx\",\n", - " names=[\"student_id\", \"full_name\", \"favourite_food\", \"meal_plan\", \"age\"],\n", - ")\n", - "students[\"age\"] = students[\"age\"].replace(\"five\", 5)\n", - "students" - ] - }, - { - "cell_type": "markdown", - "id": "c8a07159", - "metadata": {}, - "source": [ - "Okay, now we can apply the data types." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f67490d3", - "metadata": {}, - "outputs": [], - "source": [ - "students = students.astype(\n", - " {\n", - " \"student_id\": \"Int64\",\n", - " \"full_name\": \"string\",\n", - " \"favourite_food\": \"string\",\n", - " \"meal_plan\": \"category\",\n", - " \"age\": \"Int64\",\n", - " }\n", - ")\n", - "students.info()" - ] - }, - { - "cell_type": "markdown", - "id": "362ff5a5", - "metadata": {}, - "source": [ - "It took multiple steps and trial-and-error to load the data in exactly the format we want, and this is not unexpected. Data science is an iterative process. There is no way to know exactly what the data will look like until you load it and take a look at it. The general pattern we used is load the data, take a peek, make adjustments to your code, load it again, and repeat until you're happy with the result." - ] - }, - { - "cell_type": "markdown", - "id": "8b00c2a4", - "metadata": {}, - "source": [ - "### Reading Individual Sheets\n", - "\n", - "An important feature that distinguishes spreadsheets from flat files is the notion of multiple sheets. The figure below shows an Excel spreadsheet with multiple sheets. The data come from the **palmerpenguins** dataset {cite}`horst2020palmerpenguins`. Each sheet contains information on penguins from a different island where data were collected.\n", - "\n", - "![A look at the penguins spreadsheet in Excel. The spreadsheet contains has three sheets: Torgersen Island, Biscoe Island, and Dream Island.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-penguins-islands.png)\n", - "\n", - "You can read a single sheet using the following command (so as not to show the whole file, we'll use `.head()` to just show the first 5 rows):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e37f9e3d", - "metadata": {}, - "outputs": [], - "source": [ - "pd.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/penguins.xlsx\",\n", - " sheet_name=\"Torgersen Island\",\n", - ").head()" - ] - }, - { - "cell_type": "markdown", - "id": "641f6831", - "metadata": {}, - "source": [ - "Now this relies on us knowing the names of the sheets in advance. There will be situations where you can to read in data without peeking into the Excel spreadsheet. To read all sheets in, use `sheet_name=None`. The object that's created is a dictionary with key value pairs that are sheet names and data frames respectively. Let's look at the second key value pair (note that we have to convert the keys() and values() objects to list to then retrieve the second element of each using a subscript, ie `list(dictionary.keys())[]`).\n", - "\n", - "To give a sense of how this works, let's first print all of the retrieved keys:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0f3a3b78", - "metadata": {}, - "outputs": [], - "source": [ - "penguins_dict = pd.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/penguins.xlsx\",\n", - " sheet_name=None,\n", - ")\n", - "print([x for x in penguins_dict.keys()])" - ] - }, - { - "cell_type": "markdown", - "id": "076f1ebe", - "metadata": {}, - "source": [ - "Now let's show the second entry data frame" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15495426", - "metadata": {}, - "outputs": [], - "source": [ - "print(list(penguins_dict.keys())[1])\n", - "list(penguins_dict.values())[1].head()" - ] - }, - { - "cell_type": "markdown", - "id": "536ab4bb", - "metadata": {}, - "source": [ - "What we really want is these three *consistent* datasets to be in the *same* single data frame. For this, we can use the `pd.concat()` function. This concatenates any given iterable of data frames." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "151ba846", - "metadata": {}, - "outputs": [], - "source": [ - "penguins = pd.concat(penguins_dict.values(), axis=0)\n", - "penguins" - ] - }, - { - "cell_type": "markdown", - "id": "aa458187", - "metadata": {}, - "source": [ - "### Reading Part of a Sheet\n", - "\n", - "Since many use Excel spreadsheets for presentation as well as for data storage, it's quite common to find cell entries in a spreadsheet that are not part of the data you want to read in.\n", - "\n", - "The figure below shows such a spreadsheet: in the middle of the sheet is what looks like a data frame but there is extraneous text in cells above and below the data.\n", - "\n", - "![A look at the deaths spreadsheet in Excel. The spreadsheet has four rows on top that contain non-data information; the text 'For the same of consistency in the data layout, which is really a beautiful thing, I will keep making notes up here.' is spread across cells in these top four rows. Then, there is a data frame that includes information on deaths of 10 famous people, including their names, professions, ages, whether they have kids or not, date of birth and death. At the bottom, there are four more rows of non-data information; the text 'This has been really fun, but we're signing off now!' is spread across cells in these bottom four rows.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-deaths.png)\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "id": "61505c86", - "metadata": {}, - "source": [ - "This spreadsheet can be downloaded from [here](https://github.com/aeturrell/python4DS/tree/main/data) or you can load it directly from a URL. If you want to load it from your own computer's disk, you'll need to save it in a sub-folder called \"data\" first.\n", - "\n", - "\n", - "The top three rows and the bottom four rows are not part of the data frame. We could skip the top three rows with `skiprows`. Note that we set `skiprows=4` since the fourth row contains column names, not the data.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "29987b9d", - "metadata": {}, - "outputs": [], - "source": [ - "pd.read_excel(\"data/deaths.xlsx\", skiprows=4)" - ] - }, - { - "cell_type": "markdown", - "id": "a1a8c3ca", - "metadata": {}, - "source": [ - "We could also set `nrows` to omit the extraneous rows at the bottom (another option would to be to skip a set number of rows at the end using `skipfooter`)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9d7a3db3", - "metadata": {}, - "outputs": [], - "source": [ - "pd.read_excel(\"data/deaths.xlsx\", skiprows=4, nrows=10)" - ] - }, - { - "cell_type": "markdown", - "id": "2d277c3a", - "metadata": {}, - "source": [ - "## Data Types\n", - "\n", - "In CSV files, all values are strings. This is not particularly true to the data, but it is simple: everything is a string.\n", - "\n", - "The underlying data in Excel spreadsheets is more complex. A cell can be one of five things:\n", - "\n", - "- A logical, like TRUE / FALSE\n", - "\n", - "- A number, like \"10\" or \"10.5\"\n", - "\n", - "- A date, which can also include time like \"11/1/21\" or \"11/1/21 3:00 PM\"\n", - "\n", - "- A string, like \"ten\"\n", - "\n", - "- A currency, which allows numeric values in a limited range and four decimal digits of fixed precision\n", - "\n", - "When working with spreadsheet data, it's important to keep in mind that how the underlying data is stored can be very different than what you see in the cell. For example, Excel has no notion of an integer. All numbers are stored as floating points (real number), but you can choose to display the data with a customizable number of decimal points. Similarly, dates are actually stored as numbers, specifically the number of seconds since January 1, 1970. You can customize how you display the date by applying formatting in Excel. Confusingly, it's also possible to have something that looks like a number but is actually a string (e.g. type `'10` into a cell in Excel).\n", - "\n", - "These differences between how the underlying data are stored vs. how they're displayed can cause surprises when the data are loaded into analytical tools such as **pandas**. By default, **pandas** will guess the data type in a given column.\n", - "A recommended workflow is to let **pandas** guess the column types initially, inspect them, and then change any data types that you want to." - ] - }, - { - "cell_type": "markdown", - "id": "0d73bbad", - "metadata": {}, - "source": [ - "## Writing to Excel\n", - "\n", - "Let's create a small data frame that we can then write out. Note that `item` is a category and `quantity` is an integer." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15963e18", - "metadata": {}, - "outputs": [], - "source": [ - "bake_sale = pd.DataFrame(\n", - " {\"item\": pd.Categorical([\"brownie\", \"cupcake\", \"cookie\"]), \"quantity\": [10, 5, 8]}\n", - ")\n", - "bake_sale" - ] - }, - { - "cell_type": "markdown", - "id": "345bca3d", - "metadata": {}, - "source": [ - "You can write data back to disk as an Excel file using the `.to_excel()` function. The `index=False` keyword argument just writes the two columns without the index that was automatically added in the last step." - ] - }, - { - "cell_type": "markdown", - "id": "1fc17141", - "metadata": {}, - "source": [ - "```python\n", - "bake_sale.to_excel(\"data/bake_sale.xlsx\", index=False)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "f892c2e7", - "metadata": {}, - "source": [ - "The figure below shows what the data looks like in Excel.\n", - "\n", - "![Bake sale data frame created earlier in Excel.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-bake-sale.png)" - ] - }, - { - "cell_type": "markdown", - "id": "8d555c84", - "metadata": {}, - "source": [ - "Just like reading from a CSV, information on data type is lost when we read the data back in—you can see this is you read the data back in and check the `info` for the data types. Although we kept `int64` because **pandas** recognise that the second column was of integer type, we lost the categorical data type for \"item\". This data type loss makes Excel files unreliable for caching interim results." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "27e128f9", - "metadata": {}, - "outputs": [], - "source": [ - "pd.read_excel(\"data/bake_sale.xlsx\").info()" - ] - }, - { - "cell_type": "markdown", - "id": "215634bf", - "metadata": {}, - "source": [ - "### Formatted Output\n", - "\n", - "If you need more formatting options and more control over how you write spreadsheets, check out the documentation for [openpyxl](https://openpyxl.readthedocs.io/) which can do pretty much everything you imagine. Generally, releasing data in spreadsheets is not the best option: but if you do want to release data in spreadsheets according to best practice, then check out [gptables](https://gptables.readthedocs.io/)." - ] - } - ], - "metadata": { - "interpreter": { - "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" - }, - "jupytext": { - "cell_metadata_filter": "-all", - "encoding": "# -*- coding: utf-8 -*-", - "formats": "md:myst", - "main_language": "python" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.12" - }, - "toc-showtags": true - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "id": "95f0a171", + "metadata": {}, + "source": [ + "# Spreadsheets {#sec-spreadsheets}\n", + "\n", + "## Introduction\n", + "\n", + "This chapter will show you how to work with spreadsheets, for example Microsoft Excel files, in Python. We already saw how to import csv (and tsv) files in @sec-data-import. In this chapter we will introduce you to tools for working with data in Excel spreadsheets and Google Sheets.\n", + "\n", + "If you or your collaborators are using spreadsheets for organising data that will be ingested by an analytical tool like Python, we recommend reading the paper \"Data Organization in Spreadsheets\" by Karl Broman and Kara Woo {cite}`broman2018data`. The best practices presented in this paper will save you much headache down the line when you import the data from a spreadsheet into Python to analyse and visualise. (For spreadsheets that are meant to be read by humans, we recommend the [good practice tables](https://github.com/best-practice-and-impact/gptables) package.)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "51a55374", + "metadata": { + "tags": [ + "remove-cell" + ] + }, + "outputs": [], + "source": [ + "# remove cell\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib_inline.backend_inline\n", + "\n", + "# Plot settings\n", + "plt.style.use(\"https://github.com/aeturrell/python4DS/raw/main/plot_style.txt\")\n", + "matplotlib_inline.backend_inline.set_matplotlib_formats(\"svg\")" + ] + }, + { + "cell_type": "markdown", + "id": "17575f3a", + "metadata": {}, + "source": [ + "### Prerequisites\n", + "\n", + "You will need the **polars** package for this chapter. Install **fastexcel** so `read_excel()` can use the default (fast) engine, **xlsxwriter** for `write_excel()`, and **openpyxl** if you want to use the **openpyxl** engine explicitly (`uv add fastexcel xlsxwriter openpyxl`)." + ] + }, + { + "cell_type": "markdown", + "id": "ff99055a", + "metadata": {}, + "source": [ + "## Reading Excel (and Similar) Files\n", + "\n", + "**polars** can read in xls, xlsx, xlsm, xlsb, odf, ods, and odt files from your local filesystem or from a URL. It also supports an option to read a single sheet or a list of sheets. The default reader uses **fastexcel** (Rust-backed, via the **fastexcel** Python package); you can also select other engines such as **openpyxl** when you need engine-specific options.\n", + "\n", + "To show how this works, we'll work with an example spreadsheet called \"students.xlsx\". The figure below shows what the spreadsheet looks like.\n", + "\n", + "![A look at the students spreadsheet in Excel. The spreadsheet contains information on 6 students, their ID, full name, favourite food, meal plan, and age.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-students.png)" + ] + }, + { + "cell_type": "markdown", + "id": "29f2f4e0", + "metadata": {}, + "source": [ + "The first argument to `pl.read_excel()` is the path to the file to read. If you have downloaded the [file]() onto your computer and put it in a subfolder called \"data\" then you would want to use the path \"data/students.xlsx\" but we can also load it directly from the URL." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf00d4c6", + "metadata": {}, + "outputs": [], + "source": [ + "import polars as pl\n", + "\n", + "students = pl.read_excel(\n", + " \"https://github.com/aeturrell/python4DS/raw/main/data/students.xlsx\",\n", + ")\n", + "students" + ] + }, + { + "cell_type": "markdown", + "id": "03d6ba9c", + "metadata": {}, + "source": [ + "We have six students in the data and five variables on each student. However there are a few things we might want to address in this dataset:\n", + "\n", + "- The column names are all over the place. You can rename them to follow a consistent format; we recommend `snake_case`. If you want to replace **every** column name in order, assigning to `students.columns` is clear and short. If you only rename some columns, use `.rename({\"Old Name\": \"new_name\", ...})` with the exact strings from the sheet.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ecce26ea", + "metadata": {}, + "outputs": [], + "source": [ + "students.columns = [\n", + " \"student_id\",\n", + " \"full_name\",\n", + " \"favourite_food\",\n", + " \"meal_plan\",\n", + " \"age\",\n", + "]\n", + "students" + ] + }, + { + "cell_type": "markdown", + "id": "bb07ad4f", + "metadata": {}, + "source": [ + "\n", + "- `age` may be inferred as strings (for example **Utf8**) when the column mixes numeric values and text, but we want it numeric. Just like with `read_csv()`, you can supply a `schema_overrides` argument to `read_excel()` and specify Polars data types for the columns you read in (for example `pl.Int64`, `pl.Utf8`, `pl.Boolean`, `pl.Datetime`, and more). That still will not fix a value like `\"five\"` until we map it to a number first.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22e45cc2", + "metadata": {}, + "outputs": [], + "source": [ + "students = pl.read_excel(\"data/students.xlsx\")\n", + "students.columns = [\n", + " \"student_id\",\n", + " \"full_name\",\n", + " \"favourite_food\",\n", + " \"meal_plan\",\n", + " \"age\",\n", + "]\n", + "students = students.with_columns(pl.col(\"age\").replace({\"five\": 5}))\n", + "students" + ] + }, + { + "cell_type": "markdown", + "id": "c8a07159", + "metadata": {}, + "source": [ + "Okay, now we can apply the data types." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f67490d3", + "metadata": {}, + "outputs": [], + "source": [ + "students = students.with_columns(\n", + " [\n", + " pl.col(\"student_id\").cast(pl.Int64),\n", + " pl.col(\"full_name\").cast(pl.Utf8),\n", + " pl.col(\"favourite_food\").cast(pl.Utf8),\n", + " pl.col(\"meal_plan\").cast(pl.Categorical),\n", + " pl.col(\"age\").cast(pl.Int64),\n", + " ]\n", + ")\n", + "students.schema" + ] + }, + { + "cell_type": "markdown", + "id": "362ff5a5", + "metadata": {}, + "source": [ + "It took multiple steps and trial-and-error to load the data in exactly the format we want, and this is not unexpected. Data science is an iterative process. There is no way to know exactly what the data will look like until you load it and take a look at it. The general pattern we used is load the data, take a peek, make adjustments to your code, load it again, and repeat until you're happy with the result." + ] + }, + { + "cell_type": "markdown", + "id": "8b00c2a4", + "metadata": {}, + "source": [ + "### Reading Individual Sheets\n", + "\n", + "An important feature that distinguishes spreadsheets from flat files is the notion of multiple sheets. The figure below shows an Excel spreadsheet with multiple sheets. The data come from the **palmerpenguins** dataset {cite}`horst2020palmerpenguins`. Each sheet contains information on penguins from a different island where data were collected.\n", + "\n", + "![A look at the penguins spreadsheet in Excel. The spreadsheet contains has three sheets: Torgersen Island, Biscoe Island, and Dream Island.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-penguins-islands.png)\n", + "\n", + "You can read a single sheet using the following command (so as not to show the whole file, we'll use `.head()` to just show the first 5 rows):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e37f9e3d", + "metadata": {}, + "outputs": [], + "source": [ + "pl.read_excel(\n", + " \"https://github.com/aeturrell/python4DS/raw/main/data/penguins.xlsx\",\n", + " sheet_name=\"Torgersen Island\",\n", + ").head()" + ] + }, + { + "cell_type": "markdown", + "id": "641f6831", + "metadata": {}, + "source": [ + "Now this relies on us knowing the names of the sheets in advance. There will be situations where you want to read in data without peeking into the Excel spreadsheet. To read all sheets in Polars, use `sheet_id=0` (or `sheet_name=None`, which also works in recent versions of Polars). The object that’s created is a dictionary where the keys are the sheet names and the values are Polars DataFrames. To access a specific sheet, you can convert the keys() or values() to a list and then index into it, ie `list(dictionary.keys())[]` .\n", + "\n", + "To give a sense of how this works, let's first print all of the retrieved keys:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f3a3b78", + "metadata": {}, + "outputs": [], + "source": [ + "penguins_dict = pl.read_excel(\n", + " \"https://github.com/aeturrell/python4DS/raw/main/data/penguins.xlsx\",\n", + " sheet_id=0,\n", + ")\n", + "print([x for x in penguins_dict.keys()])" + ] + }, + { + "cell_type": "markdown", + "id": "076f1ebe", + "metadata": {}, + "source": [ + "Now let's show the second entry data frame" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15495426", + "metadata": {}, + "outputs": [], + "source": [ + "print(list(penguins_dict.keys())[1])\n", + "list(penguins_dict.values())[1].head()" + ] + }, + { + "cell_type": "markdown", + "id": "536ab4bb", + "metadata": {}, + "source": [ + "What we really want is these three *consistent* datasets to be in the *same* single data frame. For this, we can use the `pl.concat()` function. This concatenates any given iterable of data frames." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "151ba846", + "metadata": {}, + "outputs": [], + "source": [ + "penguins = pl.concat(penguins_dict.values())\n", + "penguins" + ] + }, + { + "cell_type": "markdown", + "id": "aa458187", + "metadata": {}, + "source": [ + "### Reading Part of a Sheet\n", + "\n", + "Since many use Excel spreadsheets for presentation as well as for data storage, it's quite common to find cell entries in a spreadsheet that are not part of the data you want to read in.\n", + "\n", + "The figure below shows such a spreadsheet: in the middle of the sheet is what looks like a data frame but there is extraneous text in cells above and below the data.\n", + "\n", + "![A look at the deaths spreadsheet in Excel. The spreadsheet has four rows on top that contain non-data information; the text 'For the same of consistency in the data layout, which is really a beautiful thing, I will keep making notes up here.' is spread across cells in these top four rows. Then, there is a data frame that includes information on deaths of 10 famous people, including their names, professions, ages, whether they have kids or not, date of birth and death. At the bottom, there are four more rows of non-data information; the text 'This has been really fun, but we're signing off now!' is spread across cells in these bottom four rows.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-deaths.png)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "61505c86", + "metadata": {}, + "source": [ + "This spreadsheet can be downloaded from [here](https://github.com/aeturrell/python4DS/tree/main/data) or you can load it directly from a URL. If you want to load it from your own computer's disk, you'll need to save it in a sub-folder called \"data\" first.\n", + "\n", + "\n", + "The top three rows and the bottom four rows are not part of the data frame. We could skip the top three rows by passing `read_options` to `read_excel()`. Note that we set `skip_rows=4` since the fourth row contains column names, not the data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29987b9d", + "metadata": {}, + "outputs": [], + "source": [ + "pl.read_excel(\n", + " \"data/deaths.xlsx\",\n", + " read_options={\"skip_rows\": 4},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "a1a8c3ca", + "metadata": {}, + "source": [ + "We could also set `n_rows` inside `read_options` to omit the extraneous rows at the bottom (another option would be to skip a set number of rows at the end using `skip_footer` in `read_options`, depending on the engine)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d7a3db3", + "metadata": {}, + "outputs": [], + "source": [ + "pl.read_excel(\n", + " \"data/deaths.xlsx\",\n", + " read_options={\"skip_rows\": 4, \"n_rows\": 10},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "2d277c3a", + "metadata": {}, + "source": [ + "## Data Types\n", + "\n", + "In CSV files, all values are strings. This is not particularly true to the data, but it is simple: everything is a string.\n", + "\n", + "The underlying data in Excel spreadsheets is more complex. A cell can be one of five things:\n", + "\n", + "- A logical, like TRUE / FALSE\n", + "\n", + "- A number, like \"10\" or \"10.5\"\n", + "\n", + "- A date, which can also include time like \"11/1/21\" or \"11/1/21 3:00 PM\"\n", + "\n", + "- A string, like \"ten\"\n", + "\n", + "- A currency, which allows numeric values in a limited range and four decimal digits of fixed precision\n", + "\n", + "When working with spreadsheet data, it's important to keep in mind that how the underlying data is stored can be very different than what you see in the cell. For example, Excel has no notion of an integer. All numbers are stored as floating points (real number), but you can choose to display the data with a customizable number of decimal points. Similarly, dates are actually stored as numbers, specifically the number of seconds since January 1, 1970. You can customize how you display the date by applying formatting in Excel. Confusingly, it's also possible to have something that looks like a number but is actually a string (e.g. type `'10` into a cell in Excel).\n", + "\n", + "These differences between how the underlying data are stored vs. how they're displayed can cause surprises when the data are loaded into analytical tools such as **polars**. By default, **polars** will guess the data type in a given column.\n", + "A recommended workflow is to let **polars** guess the column types initially, inspect them, and then change any data types that you want to." + ] + }, + { + "cell_type": "markdown", + "id": "0d73bbad", + "metadata": {}, + "source": [ + "## Writing to Excel\n", + "\n", + "Let's create a small data frame that we can then write out. Note that `item` is a category and `quantity` is an integer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15963e18", + "metadata": {}, + "outputs": [], + "source": [ + "bake_sale = pl.DataFrame(\n", + " {\n", + " \"item\": pl.Series([\"brownie\", \"cupcake\", \"cookie\"], dtype=pl.Categorical),\n", + " \"quantity\": [10, 5, 8],\n", + " }\n", + ")\n", + "bake_sale" + ] + }, + { + "cell_type": "markdown", + "id": "345bca3d", + "metadata": {}, + "source": [ + "You can write data back to disk as an Excel file using the `.write_excel()` method. Polars does not use a row index like pandas, so only the columns in the DataFrame are written by default." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1fc17141", + "metadata": {}, + "outputs": [], + "source": [ + "bake_sale.write_excel(\"data/bake_sale.xlsx\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "f892c2e7", + "metadata": {}, + "source": [ + "The figure below shows what the data looks like in Excel.\n", + "\n", + "![Bake sale data frame created earlier in Excel.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-bake-sale.png)" + ] + }, + { + "cell_type": "markdown", + "id": "8d555c84", + "metadata": {}, + "source": [ + "Just like reading from a CSV, information on data type is lost when we read the data back in—you can see this if you read the data back in and check the `schema` for the data types. Although we kept `Int64` because **polars** recognised that the second column was of integer type, we lost the categorical data type for \"item\". This data type loss makes Excel files unreliable for caching interim results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27e128f9", + "metadata": {}, + "outputs": [], + "source": [ + "pl.read_excel(\"data/bake_sale.xlsx\").schema" + ] + }, + { + "cell_type": "markdown", + "id": "215634bf", + "metadata": {}, + "source": [ + "### Formatted Output\n", + "\n", + "If you need more formatting options and more control over how you write spreadsheets, check out the documentation for [openpyxl](https://openpyxl.readthedocs.io/) which can do pretty much everything you imagine. Generally, releasing data in spreadsheets is not the best option: but if you do want to release data in spreadsheets according to best practice, then check out [gptables](https://gptables.readthedocs.io/)." + ] + } + ], + "metadata": { + "interpreter": { + "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" + }, + "jupytext": { + "cell_metadata_filter": "-all", + "encoding": "# -*- coding: utf-8 -*-", + "formats": "md:myst", + "main_language": "python" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + }, + "toc-showtags": true + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/uv.lock b/uv.lock index 8316606..8e7d9ad 100644 --- a/uv.lock +++ b/uv.lock @@ -354,6 +354,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/fd/afcd0496feca3276f509df3dbd5dae726fcc756f1a08d9e25abe1733f962/executing-2.1.0-py2.py3-none-any.whl", hash = "sha256:8d63781349375b5ebccc3142f4b30350c0cd9c79f921cde38be2be4637e98eaf", size = 25805, upload-time = "2024-09-01T12:37:33.007Z" }, ] +[[package]] +name = "fastexcel" +version = "0.19.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/c8/3b09911348e9c64dbf41096d3e8f0e93c141a23990ec9f32514111bd5f55/fastexcel-0.19.0.tar.gz", hash = "sha256:216c3719ee90963bd93a0bf8c10b177233046ac975b67651152fdaedd3c99aa1", size = 60323, upload-time = "2026-01-20T11:17:37.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/e0/3820e93ea606549cfddb8c437141dd69f2b245e74785efc8bd7511ba909d/fastexcel-0.19.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:68601072a0b4b4277c165b68f1055f88ef7ffe7ed6f08c1eeda0f0271e3f7da0", size = 3082362, upload-time = "2026-01-20T11:17:27.157Z" }, + { url = "https://files.pythonhosted.org/packages/66/0f/b42dc09515879192919942157292912393584045fd8bad98bd92961d4c30/fastexcel-0.19.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c8a87d94445678e7e3f46a6aa39d2afaee5b88a983ec3661143a6488d8955f44", size = 2864365, upload-time = "2026-01-20T11:17:28.786Z" }, + { url = "https://files.pythonhosted.org/packages/8e/4a/bc358b20fcff64b4c14ff7d7a0e1f797792b8b77e30ae755873c02362538/fastexcel-0.19.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e94fc1be6642555f277af792c22a9f80ec9b4d640d9690f00abb822b6d865069", size = 3186426, upload-time = "2026-01-20T11:17:19.087Z" }, + { url = "https://files.pythonhosted.org/packages/58/ae/d2ffdc5ad14190153e2422fc90a1052a4b0c3086d24cb8ae8967575321d8/fastexcel-0.19.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:334f9f40cd68b5924a712b6c104949757a0b8ad8a7e3fa3f3fad1c1ebc00258b", size = 3365628, upload-time = "2026-01-20T11:17:21.116Z" }, + { url = "https://files.pythonhosted.org/packages/6e/67/5f6d4e7760dc3dd8244cd124dabdd5bb7622bf1197edcc2513648847690e/fastexcel-0.19.0-cp310-abi3-win_amd64.whl", hash = "sha256:fbbdf9de79c3ef3572809bb187927c0dc5840968ffe513ea015a383024b7c6b0", size = 2905173, upload-time = "2026-01-20T11:17:33.687Z" }, +] + [[package]] name = "fastjsonschema" version = "2.21.1" @@ -1725,10 +1738,11 @@ wheels = [ [[package]] name = "python4ds" -version = "1.0.4" +version = "0.0.1" source = { virtual = "." } dependencies = [ { name = "beautifulsoup4" }, + { name = "fastexcel" }, { name = "graphviz" }, { name = "ibis-framework", extra = ["sqlite"] }, { name = "ipykernel" }, @@ -1759,12 +1773,14 @@ dependencies = [ { name = "toml" }, { name = "watermark" }, { name = "wbgapi" }, + { name = "xlsxwriter" }, { name = "yfinance" }, ] [package.metadata] requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.12.3" }, + { name = "fastexcel", specifier = ">=0.19.0" }, { name = "graphviz", specifier = ">=0.20.3" }, { name = "ibis-framework", extras = ["sqlite"], specifier = ">=9.5.0" }, { name = "ipykernel", specifier = ">=6.29.5" }, @@ -1795,6 +1811,7 @@ requires-dist = [ { name = "toml", specifier = ">=0.10.2" }, { name = "watermark", specifier = ">=2.5.0" }, { name = "wbgapi", specifier = ">=1.0.14" }, + { name = "xlsxwriter", specifier = ">=3.2.0" }, { name = "yfinance", specifier = ">=1.2.1" }, ] @@ -2463,6 +2480,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/21/02/88b65cc394961a60c43c70517066b6b679738caf78506a5da7b88ffcb643/widgetsnbextension-4.0.13-py3-none-any.whl", hash = "sha256:74b2692e8500525cc38c2b877236ba51d34541e6385eeed5aec15a70f88a6c71", size = 2335872, upload-time = "2024-08-22T12:18:19.491Z" }, ] +[[package]] +name = "xlsxwriter" +version = "3.2.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/2c/c06ef49dc36e7954e55b802a8b231770d286a9758b3d936bd1e04ce5ba88/xlsxwriter-3.2.9.tar.gz", hash = "sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c", size = 215940, upload-time = "2025-09-16T00:16:21.63Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/0c/3662f4a66880196a590b202f0db82d919dd2f89e99a27fadef91c4a33d41/xlsxwriter-3.2.9-py3-none-any.whl", hash = "sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3", size = 175315, upload-time = "2025-09-16T00:16:20.108Z" }, +] + [[package]] name = "yfinance" version = "1.2.1" From 9886acc449113061765debcb91a9619db0264a83 Mon Sep 17 00:00:00 2001 From: Uchenna Ugoh Date: Wed, 22 Apr 2026 14:44:53 +0100 Subject: [PATCH 09/34] Updated Data Visualization notebook WIP: Data Transformation --- data-transform.ipynb | 428 ++++++++++++++++++++++--------------------- data-visualise.ipynb | 181 +++++++++++------- uv.lock | 2 +- 3 files changed, 331 insertions(+), 280 deletions(-) diff --git a/data-transform.ipynb b/data-transform.ipynb index 55f2df7..eb7c6d0 100644 --- a/data-transform.ipynb +++ b/data-transform.ipynb @@ -11,7 +11,7 @@ "\n", "It's very rare that data arrive in exactly the right form you need. Often, you'll need to create some new variables or summaries, or maybe you just want to rename the variables or reorder the observations to make the data a little easier to work with.\n", "\n", - "You'll learn how to do all that (and more!) in this chapter, which will introduce you to data transformation using the **pandas** package and a new dataset on flights that departed New York City in 2013.\n", + "You'll learn how to do all that (and more!) in this chapter, which will introduce you to data transformation using the **polars** package and a new dataset on flights that departed New York City in 2013.\n", "\n", "The goal of this chapter is to give you an overview of all the key tools for transforming a data frame, a special kind of object that holds tabular data.\n", "\n", @@ -19,17 +19,17 @@ "\n", "### Prerequisites\n", "\n", - "In this chapter we'll focus on the **pandas** package, one of the most widely used tools for data science. You'll need to ensure you have **pandas** installed. To do this, you can run" + "In this chapter we'll focus on the **polars** package, one of the most widely used tools for data science. You'll need to ensure you have **polars** installed. To do this, you can run\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "04dcb195", "metadata": {}, "outputs": [], "source": [ - "import pandas as pd" + "import polars as pl" ] }, { @@ -37,9 +37,9 @@ "id": "438cc0a4", "metadata": {}, "source": [ - "If this command fails, you don't have **pandas** installed. Open up the terminal in Visual Studio Code (Terminal -> New Terminal), `cd` to the folder you are working in, and type in `uv add pandas`.\n", + "If this command fails, you don't have **polars** installed. Open up the terminal in Visual Studio Code (Terminal -> New Terminal), `cd` to the folder you are working in, and type in `uv add polars`.\n", "\n", - "Furthermore, if you wish to check which version of **pandas** you're using, it's" + "Furthermore, if you wish to check which version of **polars** you're using, it's\n" ] }, { @@ -49,7 +49,7 @@ "metadata": {}, "outputs": [], "source": [ - "pd.__version__" + "pl.__version__" ] }, { @@ -57,22 +57,33 @@ "id": "0c5e5b82", "metadata": {}, "source": [ - "You'll also need the data. Most of the time, data will need to be loaded from a file or the internet. These data are no different, but one of the amazing things about **pandas** is how many different types of data it can load, including from files on the internet.\n", + "You'll also need the data. Most of the time, data will need to be loaded from a file or the internet. These data are no different, but one of the amazing things about **polars** is how many different types of data it can load, including from files on the internet.\n", "\n", "The data is around 50MB in size so you will need a good internet connection or a little patience for it to download.\n", "\n", - "Let's download the data:" + "Let's download the data:\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "eff283e8", "metadata": {}, "outputs": [], "source": [ + "import io\n", + "import requests\n", + "\n", "url = \"https://raw.githubusercontent.com/byuidatascience/data4python4ds/master/data-raw/flights/flights.csv\"\n", - "flights = pd.read_csv(url)" + "resp = requests.get(url, timeout=60)\n", + "resp.raise_for_status()\n", + "\n", + "flights = pl.read_csv(\n", + " io.BytesIO(resp.content),\n", + " null_values=[\"NA\"],\n", + " truncate_ragged_lines=True,\n", + " ignore_errors=True,\n", + ")" ] }, { @@ -80,7 +91,7 @@ "id": "2907635c", "metadata": {}, "source": [ - "If the above code worked, then you've downloaded the data in CSV format and put it in a data frame. Let's look at the first few rows using the `.head()` function that works on all **pandas** data frames." + "If the above code worked, then you've downloaded the data in CSV format and put it in a data frame. Let's look at the first few rows using the `.head()` function that works on all **polars** data frames.\n" ] }, { @@ -98,7 +109,7 @@ "id": "68aada55", "metadata": {}, "source": [ - "To get more general information on the columns, the data types (`dtypes`) of the columns, and the size of the dataset, use `.info()`." + "To get more general information on the columns, the data types (`dtypes`) of the columns, and the size of the dataset, use `.glimpse()`.\n" ] }, { @@ -108,7 +119,7 @@ "metadata": {}, "outputs": [], "source": [ - "flights.info()" + "flights.glimpse(max_items_per_column=5)" ] }, { @@ -116,22 +127,23 @@ "id": "100189b8", "metadata": {}, "source": [ - "You might have noticed the short abbreviations that appear in the `Dtypes` column. These tell you the type of the values in their respective columns: `int64` is short for integer (eg whole numbers) and `float64` is short for double-precision floating point number (these are real numbers). `object` is a bit of a catch all category for any data type that **pandas** is not really confident about inferring. Although not found here, other data types include `string` for text and `datetime` for combinations of a date and time.\n", + "You might have noticed the short abbreviations that appear in the `dtype` column. These tell you the type of the values in their respective columns: `i64` is short for integer (eg whole numbers) and `f64` is short for double-precision floating point number (these are real numbers). **polars** has an `object` data type which allows storing arbitrary Python objects, but this makes you lose performance benefits, as **polars** is **strictly typed**. Although not found here, other data types include `str` for text and `datetime` for combinations of a date and time.\n", "\n", "The table below gives some of the most common data types you are likely to encounter.\n", "\n", - "| **Name of data type** | **Type of data** |\n", - "|:----------:|:-------------:|\n", - "| float64 | real numbers |\n", - "| category | categories |\n", - "| datetime64 | date times |\n", - "| int64 | integers |\n", - "| bool | True or False |\n", - "| string | text |\n", + "| **Name of data type** | **Type of data** |\n", + "| :-------------------: | :--------------: |\n", + "| Float64 | real numbers |\n", + "| Categorical | categories |\n", + "| Datetime | date and time |\n", + "| Date | date |\n", + "| Int64 | integers |\n", + "| Boolean | True or False |\n", + "| String | text |\n", "\n", "The different column data types are important because the operations you can perform on a column depend so much on its \"type\"; for example, you can remove all punctuation from strings while you can multiply ints and floats.\n", "\n", - "We would like to work with the `\"time_hour\"` variable in the form of a datetime; fortunately, **pandas** makes it easy to perform that conversion on that specific column" + "We would like to work with the `\"time_hour\"` variable in the form of a datetime; fortunately, **polars** makes it easy to perform that conversion on that specific column\n" ] }, { @@ -141,7 +153,7 @@ "metadata": {}, "outputs": [], "source": [ - "flights[\"time_hour\"]" + "flights.get_column(\"time_hour\")" ] }, { @@ -151,7 +163,9 @@ "metadata": {}, "outputs": [], "source": [ - "flights[\"time_hour\"] = pd.to_datetime(flights[\"time_hour\"], format=\"%Y-%m-%dT%H:%M:%SZ\")" + "flights.with_columns(\n", + " pl.col(\"time_hour\").str.to_datetime()\n", + ")" ] }, { @@ -159,17 +173,17 @@ "id": "6dc43cee", "metadata": {}, "source": [ - "## **pandas** basics\n", - "\n", - "**pandas** is a really comprehensive package, and this book will barely scratch the surface of what it can do. But it's built around a few simple ideas that, once they've clicked, make life a lot easier.\n", + "## **polars** basics\n", "\n", - "Let’s start with the absolute basics. The most basic pandas object is DataFrame. A DataFrame is a 2-dimensional data structure that can store data of different types (including characters, integers, floating point values, categorical data, even lists) in columns. It is made up of rows and columns (with each row-column cell containing a value), plus two bits of contextual information: the index (which carries information about each row) and the column names (which carry information about each column).\n", + "**polars** is a really comprehensive package, and this book will barely scratch the surface of what it can do. But it's built around a few simple ideas that, once they've clicked, make life a lot easier.\n", "\n", - "![](https://pandas.pydata.org/docs/_images/01_table_dataframe.svg)\n", + "Let’s start with the absolute basics. The most basic polars object is DataFrame. A DataFrame is a 2-dimensional data structure that can store data of different types (including characters, integers, floating point values, categorical data, even lists) in columns. It is made up of rows and columns (with each row-column cell containing a value), plus contextual information: column names (which carry information about each column).\n", "\n", - "Perhaps the most important notion to have about **pandas** data frames is that they are built around an index that sits on the left-hand side of the data frame. Every time you perform an operation on a data frame, you need to think about how it might or might not affect the index; or, put another way, whether you want to modify the index.\n", + "::: {.callout-note}\n", + "Note: If you're coming from **pandas**, be aware that **polars** does not use an index column and each row is indexed by its integer position in the table.\n", + ":::\n", "\n", - "Let's see a simple example of this with a made-up data frame:" + "![](https://pandas.pydata.org/docs/_images/01_table_dataframe.svg)\n" ] }, { @@ -179,7 +193,7 @@ "metadata": {}, "outputs": [], "source": [ - "df = pd.DataFrame(\n", + "df = pl.DataFrame(\n", " data={\n", " \"col0\": [0, 0, 0, 0],\n", " \"col1\": [0, 0, 0, 0],\n", @@ -187,7 +201,6 @@ " \"col3\": [\"a\", \"b\", \"b\", \"a\"],\n", " \"col4\": [\"alpha\", \"gamma\", \"gamma\", \"gamma\"],\n", " },\n", - " index=[\"row\" + str(i) for i in range(4)],\n", ")\n", "df.head()" ] @@ -197,7 +210,7 @@ "id": "185ba56e", "metadata": {}, "source": [ - "You can see there are 5 columns (named `\"col0\"` to `\"col4\"`) and that the index consists of four entries named `\"row0\"` to `\"row3\"`." + "You can see there are 5 columns (named `\"col0\"` to `\"col4\"`).\n" ] }, { @@ -205,14 +218,14 @@ "id": "3f325661", "metadata": {}, "source": [ - "A second key point you should know is that the operations on a **pandas** data frame can be chained together. We need not perform one assignment per line of code; we can actually do multiple assignments in a single command.\n", + "A second key point you should know is that the operations on a **polars** data frame can be chained together. We need not perform one assignment per line of code; we can actually do multiple assignments in a single command.\n", "\n", "Let's see an example of this. We're going to string together four operations:\n", "\n", - "1. we will use `query()` to find only the rows where the destination `\"dest\"` column has the value `\"IAH\"`. This doesn't change the index, it only removes irrelevant rows. In effect, this step removes rows we're not interested in.\n", - "2. we will use `groupby()` to group rows by the year, month, and day (we pass a list of columns to the `groupby()` function). This step changes the index; the new index will have three columns in that track the year, month, and day. In effect, this step changes the index.\n", - "3. we will choose which columns we wish to keep after the `groupby()` operation by passing a list of them to a set of square brackets (the double brackets are because it's a list within a data frame). Here we just want one column, `\"arr_delay\"`. This doesn't affect the index. In effect, this step removes columns we're not interested in.\n", - "4. finally, we must specify what `groupby()` operation we wish to apply; when aggregating the information in multiple rows down to one row, we need to say how that information should be aggregated. In this case, we'll use the `mean()`. In effect, this step applies a statistic to the variable(s) we selected earlier, across the groups we created earlier." + "1. we will use `filter()` to find only the rows where the destination `\"dest\"` column has the value `\"IAH\"`. We use the `pl.col()` expression to select the column for filtering condition. In effect, this step removes rows we're not interested in.\n", + "2. we will use `group_by()` to group rows by the year, month, and day (we pass a list of columns to the `group_by()` function).\n", + "3. we will choose which columns to perform aggregation on after the `group_by()` operation by using the `pl.col()` expression inside `agg()`, to select the column. Here we just want one column, `\"arr_delay\"`. In effect, this step removes columns we're not interested in.\n", + "4. finally, we must specify what `agg()` operation we wish to apply; when aggregating the information in multiple rows down to one row, we need to say how that information should be aggregated. In this case, we'll use the `mean()`. In effect, this step applies a statistic to the variable(s) we selected earlier, across the groups we created earlier.\n" ] }, { @@ -222,7 +235,7 @@ "metadata": {}, "outputs": [], "source": [ - "(flights.query(\"dest == 'IAH'\").groupby([\"year\", \"month\", \"day\"])[[\"arr_delay\"]].mean())" + "flights.filter(pl.col(\"dest\") == \"IAH\").group_by([\"year\", \"month\", \"day\"]).agg(pl.col(\"arr_delay\").mean())" ] }, { @@ -230,16 +243,15 @@ "id": "b8b85551", "metadata": {}, "source": [ - "You can see here that we've created a new data frame with a new index. To do it, we used four key operations:\n", + "You can see here that we've created a new data frame. To do it, we used three key operations:\n", "\n", "1. manipulating rows\n", - "2. manipulating the index\n", - "3. manipulating columns\n", - "4. applying statistics\n", + "2. manipulating columns\n", + "3. applying statistics\n", "\n", "Most operations you could want to do to a single data frame are covered by these, but there are different options for each of them depending on what you need.\n", "\n", - "Let's now dig a bit more into these operations." + "Let's now dig a bit more into these operations.\n" ] }, { @@ -249,25 +261,56 @@ "source": [ "## Manipulating Rows in Data Frames\n", "\n", - "Let's create some fake data to show how this works." + "Let's create some fake data to show how this works.\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "3958ddb5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 7)
col0col1col2col3col4col5col6
i64i64i64i64i64i64str
012345"apple"
67891011"orange"
121314151617"pineapple"
181920212223"mango"
242526272829"kiwi"
303132333435"lemon"
" + ], + "text/plain": [ + "shape: (6, 7)\n", + "┌──────┬──────┬──────┬──────┬──────┬──────┬───────────┐\n", + "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 ┆ col6 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", + "╞══════╪══════╪══════╪══════╪══════╪══════╪═══════════╡\n", + "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ 4 ┆ 5 ┆ apple │\n", + "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ 10 ┆ 11 ┆ orange │\n", + "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ 16 ┆ 17 ┆ pineapple │\n", + "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ 22 ┆ 23 ┆ mango │\n", + "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 ┆ kiwi │\n", + "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ 34 ┆ 35 ┆ lemon │\n", + "└──────┴──────┴──────┴──────┴──────┴──────┴───────────┘" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import numpy as np\n", "\n", - "df = pd.DataFrame(\n", + "df = pl.DataFrame(\n", " data=np.reshape(range(36), (6, 6)),\n", - " index=[\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"],\n", - " columns=[\"col\" + str(i) for i in range(6)],\n", - " dtype=float,\n", + " schema=[\"col\" + str(i) for i in range(6)],\n", ")\n", - "df[\"col6\"] = [\"apple\", \"orange\", \"pineapple\", \"mango\", \"kiwi\", \"lemon\"]\n", + "df.insert_column(6, pl.Series(\"col6\", [\"apple\", \"orange\", \"pineapple\", \"mango\", \"kiwi\", \"lemon\"]))\n", "df" ] }, @@ -278,9 +321,10 @@ "source": [ "### Accessing Rows\n", "\n", - "To access a particular row directly, you can use `df.loc['rowname']` or `df.loc[['rowname1', 'rowname2']]` for two different rows.\n", + "To access a particular row directly, you can get that by index (location in the data) or predicate/expression using `.row()`, which returns as a tuple.\n", + "Remember that Python indices begin from zero, so to retrieve the first row by index you would use `.row(0)`:\n", "\n", - "For example," + "For example,\n" ] }, { @@ -290,7 +334,11 @@ "metadata": {}, "outputs": [], "source": [ - "df.loc[[\"a\", \"b\"]]" + "# Gets the first row of the DataFrame\n", + "df.row(0)\n", + "\n", + "# Gets the fifth row of the DataFrame\n", + "df.row(4)" ] }, { @@ -298,7 +346,7 @@ "id": "18124edd", "metadata": {}, "source": [ - "But you can also access particular rows based on their location in the data frame using `.iloc`. Remember that Python indices begin from zero, so to retrieve the first row you would use `.iloc[0]`:\n" + "You can also access particular rows based on a predicate using `.row()` with the `by_predicate` parameter.\n" ] }, { @@ -308,7 +356,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.iloc[0]" + "df.row(by_predicate=pl.col(\"col6\") == \"mango\")" ] }, { @@ -316,7 +364,7 @@ "id": "ca822472", "metadata": {}, "source": [ - "This works for multiple rows too. Let's grab the first and third rows (in positions 0 and 2) by passing a list of positions:" + "To get the row as a dictionary instead of a tuple with a mapping of column names to row values, specify `named=True`\n" ] }, { @@ -326,15 +374,11 @@ "metadata": {}, "outputs": [], "source": [ - "df.iloc[[0, 2]]" - ] - }, - { - "cell_type": "markdown", - "id": "381eb34d", - "metadata": {}, - "source": [ - "There are other ways to access multiple rows that make use of *slicing* but we'll leave that topic for another time." + "# Get the first row of the DataFrame as a dictionary\n", + "df.row(0, named=True)\n", + "\n", + "# Get the row where col6 is \"mango\" as a dictionary\n", + "df.row(by_predicate=pl.col(\"col6\") == \"mango\", named=True)" ] }, { @@ -342,9 +386,9 @@ "id": "77f67ac2", "metadata": {}, "source": [ - "### Filtering rows with query\n", + "### Filtering rows\n", "\n", - "As with the flights example, we can also filter rows based on a condition using `query()`:" + "As with the flights example, we can also filter rows based on a condition using `filter()`:\n" ] }, { @@ -354,7 +398,9 @@ "metadata": {}, "outputs": [], "source": [ - "df.query(\"col6 == 'kiwi' or col6 == 'pineapple'\")" + "df.filter(\n", + " (pl.col(\"col6\") == \"kiwi\") | (pl.col(\"col6\") == \"pineapple\")\n", + ")" ] }, { @@ -362,7 +408,7 @@ "id": "000eb292", "metadata": {}, "source": [ - "For numbers, you can also use the greater than and less than signs:" + "For numbers, you can also use the greater than and less than signs:\n" ] }, { @@ -372,7 +418,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.query(\"col0 > 6\")" + "df.filter(pl.col(\"col0\") > 6)" ] }, { @@ -380,7 +426,7 @@ "id": "f5e03f63", "metadata": {}, "source": [ - "In fact, there are lots of options that work with `query()`: as well as `>` (greater than), you can use `>=` (greater than or equal to), `<` (less than), `<=` (less than or equal to), `==` (equal to), and `!=` (not equal to). You can also use the commands `and` as well as `or` to combine multiple conditions. Here's an example of `and` from the `flights` data frame:" + "In fact, there are lots of options that work with `filter()`: as well as `>` (greater than), you can use `>=` (greater than or equal to), `<` (less than), `<=` (less than or equal to), `==` (equal to), and `!=` (not equal to). You can also use operators `&` as well as `|` to combine multiple conditions. Here's an example of `&` from the `flights` data frame:\n" ] }, { @@ -391,7 +437,7 @@ "outputs": [], "source": [ "# Flights that departed on January 1\n", - "flights.query(\"month == 1 and day == 1\")" + "flights.filter((pl.col(\"month\") == 1) & (pl.col(\"day\") <= 5))" ] }, { @@ -399,7 +445,7 @@ "id": "bd0af6fc", "metadata": {}, "source": [ - "Note that equality is tested by `==` and *not* by `=`, because the latter is used for assignment." + "Note that equality is tested by `==` and _not_ by `=`, because the latter is used for assignment.\n" ] }, { @@ -409,7 +455,17 @@ "source": [ "### Re-arranging Rows\n", "\n", - "Again and again, you will want to re-order the rows of your data frame according to the values in a particular column. **pandas** makes this very easy via the `.sort_values()` function. It takes a data frame and a set of column names to sort by. If you provide more than one column name, each additional column will be used to break ties in the values of preceding columns. For example, the following code sorts by the departure time, which is spread over four columns." + "Again and again, you will want to re-order the rows of your data frame according to the values in a particular column. **polars** makes this very easy via the `.sort()` function. You can sort by single or multiple column names and also by expressions. If you provide more than one column name, each additional column will be used to break ties in the values of preceding columns. For example, the following code sorts by the departure time, which is spread over four columns.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "395c9c62", + "metadata": {}, + "outputs": [], + "source": [ + "flights.sort(\"dep_time\")" ] }, { @@ -419,7 +475,11 @@ "metadata": {}, "outputs": [], "source": [ - "flights.sort_values([\"year\", \"month\", \"day\", \"dep_time\"])" + "# Sort by multiple columns by passing a list of columns.\n", + "flights.sort([\"year\", \"month\", \"day\", \"dep_time\"])\n", + "\n", + "# Or use positional arguments to sort by multiple columns in the same way.\n", + "flights.sort(\"year\", \"month\", \"day\", \"dep_time\")" ] }, { @@ -427,8 +487,8 @@ "id": "39a6e9b1", "metadata": {}, "source": [ - "You can use the keyword argument `ascending=False` to re-order by a column or columns in descending order.\n", - "For example, this code shows the most delayed flights:" + "You can use the keyword argument `descending=True` to re-order by a column or columns in descending order.\n", + "For example, this code shows the most delayed flights:\n" ] }, { @@ -438,7 +498,17 @@ "metadata": {}, "outputs": [], "source": [ - "flights.sort_values(\"dep_delay\", ascending=False)" + "flights.sort(\"dep_delay\", descending=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80bf3df7", + "metadata": {}, + "outputs": [], + "source": [ + "flights.sort([\"dep_delay\", \"arr_delay\"], descending=[True, False])" ] }, { @@ -447,7 +517,7 @@ "metadata": {}, "source": [ "You can of course combine all of the above row manipulations to solve more complex problems.\n", - "For example, we could look for the top three destinations of the flights that were most delayed on arrival that left on roughly on time:" + "For example, we could look for the top three destinations of the flights that were most delayed on arrival that left on roughly on time:\n" ] }, { @@ -458,9 +528,9 @@ "outputs": [], "source": [ "(\n", - " flights.query(\"dep_delay <= 10 and dep_delay >= -10\")\n", - " .sort_values(\"arr_delay\", ascending=False)\n", - " .iloc[[0, 1, 2]]\n", + " flights.filter(pl.col(\"dep_delay\") <= 10) & (pl.col(\"dep_delay\") >= -10)\n", + " .sort(\"arr_delay\", descending=True)\n", + " .head(3)\n", ")" ] }, @@ -485,13 +555,13 @@ "\n", " f. Were delayed by at least an hour, but made up over 30 minutes in flight\n", "\n", - "2. Sort `flights` to find the flights with longest departure delays.\n", + "2. Sort `flights` to find the flights with longest departure delays.\n", "\n", - "3. Sort `flights` to find the fastest flights\n", + "3. Sort `flights` to find the fastest flights\n", "\n", - "4. Which flights traveled the farthest?\n", + "4. Which flights traveled the farthest?\n", "\n", - "5. Does it matter what order you used `query()` and `sort_values()` in if you're using both? Why/why not? Think about the results and how much work the functions would have to do." + "5. Does it matter what order you used `filter()` and `sort()` in if you're using both? Why/why not? Think about the results and how much work the functions would have to do.\n" ] }, { @@ -505,7 +575,7 @@ "\n", "::: {.callout-note}\n", "Some **pandas** operations can apply either to columns or rows, depending on the syntax used. For example, accessing values by position can be achieved in the same way for rows and columns via `.iloc` where to access the ith row you would use `df.iloc[i]` and to access the jth column you would use `df.iloc[:, j]` where `:` stands in for 'any row'.\n", - ":::" + ":::\n" ] }, { @@ -515,7 +585,7 @@ "source": [ "### Creating New Columns\n", "\n", - "Let's now move on to creating new columns, either using new information or from existing columns. Given a data frame, `df`, creating a new column with the same value repeated is as easy as using square brackets with a string (text enclosed by quotation marks) in." + "Let's now move on to creating new columns, either using new information or from existing columns. Given a data frame, `df`, creating a new column with the same value repeated is as easy as using square brackets with a string (text enclosed by quotation marks) in.\n" ] }, { @@ -525,7 +595,7 @@ "metadata": {}, "outputs": [], "source": [ - "df[\"new_column0\"] = 5\n", + "df = df.with_columns(pl.lit(5).alias(\"new_column0\"))\n", "df" ] }, @@ -534,7 +604,7 @@ "id": "55bc84a5", "metadata": {}, "source": [ - "If we do the same operation again, but with a different right-hand side, it will overwrite what was already in that column. Let's see this with an example where we put different values in each position by assigning a list to the new column." + "If we do the same operation again, but with a different right-hand side, it will overwrite what was already in that column. Let's see this with an example where we put different values in each position by assigning a list to the new column.\n" ] }, { @@ -544,7 +614,7 @@ "metadata": {}, "outputs": [], "source": [ - "df[\"new_column0\"] = [0, 1, 2, 3, 4, 5]\n", + "df = df.with_columns(new_column0=pl.Series([0, 1, 2, 3, 4, 5]))\n", "df" ] }, @@ -557,7 +627,7 @@ "What happens if you try to use assignment where the right-hand side values are longer or shorter than the length of the data frame?\n", ":::\n", "\n", - "By passing a list within the square brackets, we can actually create more than one new column:" + "By passing a list within the square brackets, we can actually create more than one new column:\n" ] }, { @@ -567,7 +637,7 @@ "metadata": {}, "outputs": [], "source": [ - "df[[\"new_column1\", \"new_column2\"]] = [5, 6]\n", + "df = df.with_columns(new_column1=pl.lit(5), new_column2=pl.lit(6))\n", "df" ] }, @@ -576,7 +646,7 @@ "id": "10792ddd", "metadata": {}, "source": [ - "Very often, you will want to create a new column that is the result of an operation on existing columns. There are a couple of ways to do this. The 'stand-alone' method works in a similar way to what we've just seen except that we refer to the data frame on the right-hand side of the assignment statement too:" + "Very often, you will want to create a new column that is the result of an operation on existing columns. There are a couple of ways to do this. The 'stand-alone' method works in a similar way to what we've just seen except that we refer to the data frame on the right-hand side of the assignment statement too:\n" ] }, { @@ -586,7 +656,7 @@ "metadata": {}, "outputs": [], "source": [ - "df[\"new_column3\"] = df[\"col0\"] - df[\"new_column0\"]\n", + "df = df.with_columns(new_column3=pl.col(\"col0\") - pl.col(\"new_column0\"))\n", "df" ] }, @@ -595,7 +665,7 @@ "id": "03172fa9", "metadata": {}, "source": [ - "The other way to do this involves an 'assign()' statement and is used when you wish to chain multiple steps together (like we saw earlier). These use a special syntax called a 'lambda' statement, which (here at least) just provides a way of specifying to **pandas** that we wish to perform the operation on every row. Below is an example using the flights data. You should note though that the word 'row' below is a dummy; you could replace it with any variable name (for example, `x`) but `row` makes what is happening a little bit clearer." + "The other way to do this involves an 'assign()' statement and is used when you wish to chain multiple steps together (like we saw earlier). These use a special syntax called a 'lambda' statement, which (here at least) just provides a way of specifying to **pandas** that we wish to perform the operation on every row. Below is an example using the flights data. You should note though that the word 'row' below is a dummy; you could replace it with any variable name (for example, `x`) but `row` makes what is happening a little bit clearer.\n" ] }, { @@ -605,27 +675,11 @@ "metadata": {}, "outputs": [], "source": [ - "(\n", - " flights.assign(\n", - " gain=lambda row: row[\"dep_delay\"] - row[\"arr_delay\"],\n", - " speed=lambda row: row[\"distance\"] / row[\"air_time\"] * 60,\n", - " )\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "c531df3e", - "metadata": {}, - "source": [ - "::: {.callout-note}\n", - "A lambda function is like any normal function in Python except that it has no name, and it tends to be contained in one line of code. A lambda function is made of an argument, a colon, and an expression, like the following lambda function that multiplies an input by three.\n", "\n", - "```python\n", - "lambda x: x*3\n", - "```\n", - "\n", - ":::" + "flights.with_columns(\n", + " (pl.col(\"dep_delay\") - pl.col(\"arr_delay\")).alias(\"gain\"),\n", + " (pl.col(\"distance\") / pl.col(\"air_time\") * 60).alias(\"speed\")\n", + ")" ] }, { @@ -633,8 +687,7 @@ "id": "82a97330", "metadata": {}, "source": [ - "### Accessing Columns\n", - "\n" + "### Accessing Columns\n" ] }, { @@ -642,7 +695,7 @@ "id": "7599db58", "metadata": {}, "source": [ - "Just as with selecting rows, there are many options and ways to select the columns to operate on. The one with the simplest syntax is the name of the data frame followed by square brackets and the column name (as a string)" + "Just as with selecting rows, there are many options and ways to select the columns to operate on. The one with the simplest syntax is the name of the data frame followed by square brackets and the column name (as a string)\n" ] }, { @@ -660,7 +713,7 @@ "id": "63bca028", "metadata": {}, "source": [ - "If you need to select *multiple* columns, you cannot just pass a string into `df[...]`; instead you need to pass an object that is iterable (and so have multiple items). The most straight forward way to select multiple columns is to pass a *list*. Remember, lists comes in square brackets so we're going to see something with repeated square brackets: one for accessing the data frame's innards and one for the list." + "If you need to select _multiple_ columns, you cannot just pass a string into `df[...]`; instead you need to pass an object that is iterable (and so have multiple items). The most straight forward way to select multiple columns is to pass a _list_. Remember, lists comes in square brackets so we're going to see something with repeated square brackets: one for accessing the data frame's innards and one for the list.\n" ] }, { @@ -678,7 +731,7 @@ "id": "2b2a7be0", "metadata": {}, "source": [ - "If you want to access particular rows at the same time, use the `.loc` access function:" + "If you want to access particular rows at the same time, use the `.loc` access function:\n" ] }, { @@ -696,7 +749,7 @@ "id": "c1b7db13", "metadata": {}, "source": [ - "And, just as with rows, we can access columns by their position using `.iloc` (where `:` stands in for 'any row')." + "And, just as with rows, we can access columns by their position using `.iloc` (where `:` stands in for 'any row').\n" ] }, { @@ -714,7 +767,7 @@ "id": "509dc236", "metadata": {}, "source": [ - "There are other ways to access multiple columns that make use of slicing but we’ll leave that topic for another time." + "There are other ways to access multiple columns that make use of slicing but we’ll leave that topic for another time.\n" ] }, { @@ -722,7 +775,7 @@ "id": "17b928c8", "metadata": {}, "source": [ - "Sometimes, you'll want to select columns based on the *type* of data that they hold. For this, **pandas** provides a function `.select_dtypes()`. Let's use this to select all columns with integers in the flights data." + "Sometimes, you'll want to select columns based on the _type_ of data that they hold. For this, **pandas** provides a function `.select_dtypes()`. Let's use this to select all columns with integers in the flights data.\n" ] }, { @@ -740,9 +793,9 @@ "id": "9aec778c", "metadata": {}, "source": [ - "There are other occasions when you'd like to select columns based on criteria such as patterns in the *name* of the column. Because Python has very good support for text, this is very possible but doesn't tend to be so built-in to **pandas** functions. The trick is to generate a list of column names that you want from the pattern you're interested in.\n", + "There are other occasions when you'd like to select columns based on criteria such as patterns in the _name_ of the column. Because Python has very good support for text, this is very possible but doesn't tend to be so built-in to **pandas** functions. The trick is to generate a list of column names that you want from the pattern you're interested in.\n", "\n", - "Let's see a couple of examples. First, let's get all columns in our `df` data frame that begin with `\"new_...\"`. We'll generate a list of true and false values reflecting if each of the columns begins with \"new\" and then we'll pass those true and false values to `.loc`, which will only give columns for which the result was `True`. To show what's going on, we'll break it into two steps:" + "Let's see a couple of examples. First, let's get all columns in our `df` data frame that begin with `\"new_...\"`. We'll generate a list of true and false values reflecting if each of the columns begins with \"new\" and then we'll pass those true and false values to `.loc`, which will only give columns for which the result was `True`. To show what's going on, we'll break it into two steps:\n" ] }, { @@ -769,7 +822,7 @@ "id": "b514cbf4", "metadata": {}, "source": [ - "As well as `startswith()`, there are other commands like `endswith()`, `contains()`, `isnumeric()`, and `islower()`." + "As well as `startswith()`, there are other commands like `endswith()`, `contains()`, `isnumeric()`, and `islower()`.\n" ] }, { @@ -779,7 +832,7 @@ "source": [ "### Renaming Columns\n", "\n", - "There are three easy ways to rename columns, depending on what the context is. The first is to use the dedicated `rename()` function with an object called a dictionary. Dictionaries in Python consist of curly brackets with comma separated pairs of values where the first values maps into the second value. An example of a dictionary would be `{'old_col1': 'new_col1', 'old_col2': 'new_col2'}`. Let's see this in practice (but note that we are not 'saving' the resulting data frame, just showing it—to save it, you'd need to add `df = ` to the left-hand side of the code below)." + "There are three easy ways to rename columns, depending on what the context is. The first is to use the dedicated `rename()` function with an object called a dictionary. Dictionaries in Python consist of curly brackets with comma separated pairs of values where the first values maps into the second value. An example of a dictionary would be `{'old_col1': 'new_col1', 'old_col2': 'new_col2'}`. Let's see this in practice (but note that we are not 'saving' the resulting data frame, just showing it—to save it, you'd need to add `df = ` to the left-hand side of the code below).\n" ] }, { @@ -797,7 +850,7 @@ "id": "0a673852", "metadata": {}, "source": [ - "The second method is for when you want to rename all of the columns. For that you simply set `df.columns` equal to the new set of columns that you'd like to have. For example, we might want to capitalise the first letter of each column using `str.capitalize()` and assign that to `df.columns`." + "The second method is for when you want to rename all of the columns. For that you simply set `df.columns` equal to the new set of columns that you'd like to have. For example, we might want to capitalise the first letter of each column using `str.capitalize()` and assign that to `df.columns`.\n" ] }, { @@ -816,7 +869,7 @@ "id": "7a8b9660", "metadata": {}, "source": [ - "Finally, we might be interested in just replacing specific parts of column names. In this case, we can use `.str.replace()`. As an example, let's add the word `\"Original\"` ahead of the original columns:" + "Finally, we might be interested in just replacing specific parts of column names. In this case, we can use `.str.replace()`. As an example, let's add the word `\"Original\"` ahead of the original columns:\n" ] }, { @@ -834,7 +887,7 @@ "id": "09632b99", "metadata": {}, "source": [ - "### Re-ordering Columns" + "### Re-ordering Columns\n" ] }, { @@ -844,9 +897,9 @@ "source": [ "By default, new columns are added to the right-hand side of the data frame. But you may have reasons to want the columns to appear in a particular order, or perhaps you'd just find it more convenient to have new columns on the left-hand side when there are many columns in a data frame (which happens a lot).\n", "\n", - "The simplest way to re-order (all) columns is to create a new list of their names with them in the order that you'd like them: but be careful you don't forget any columns that you'd like to keep! \n", + "The simplest way to re-order (all) columns is to create a new list of their names with them in the order that you'd like them: but be careful you don't forget any columns that you'd like to keep!\n", "\n", - "Let's see an example with a fresh version of the fake data from earlier. We'll put all of the odd-numbered columns first, in descending order, then the even similarly." + "Let's see an example with a fresh version of the fake data from earlier. We'll put all of the odd-numbered columns first, in descending order, then the even similarly.\n" ] }, { @@ -856,7 +909,7 @@ "metadata": {}, "outputs": [], "source": [ - "df = pd.DataFrame(\n", + "df = pl.DataFrame(\n", " data=np.reshape(range(36), (6, 6)),\n", " index=[\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"],\n", " columns=[\"col\" + str(i) for i in range(6)],\n", @@ -881,7 +934,7 @@ "id": "dd91d87a", "metadata": {}, "source": [ - "Of course, this is quite tedious if you have lots of columns! There are methods that can help make this easier depending on your context. Perhaps you'd just liked to sort the columns in order? This can be achieved by combining `sorted()` and the `reindex()` command (which works for rows or columns) with `axis=1`, which means the second axis (i.e. columns)." + "Of course, this is quite tedious if you have lots of columns! There are methods that can help make this easier depending on your context. Perhaps you'd just liked to sort the columns in order? This can be achieved by combining `sorted()` and the `reindex()` command (which works for rows or columns) with `axis=1`, which means the second axis (i.e. columns).\n" ] }, { @@ -920,7 +973,7 @@ "- `df.iloc[0, 0]`\n", "- `df.loc[\"row\", \"column\"]`\n", "\n", - "In the above examples, square brackets are instructions about *where* to grab bits from the data frame. They are a bit like an address system for values within a data frame. Square brackets *also* denote lists though. So if you want to select *multiple* columns or rows, you might see syntax like this:\n", + "In the above examples, square brackets are instructions about _where_ to grab bits from the data frame. They are a bit like an address system for values within a data frame. Square brackets _also_ denote lists though. So if you want to select _multiple_ columns or rows, you might see syntax like this:\n", "\n", "`df.loc[[\"row0\", \"row1\"], [\"column0\", \"column2\"]]`\n", "\n", @@ -963,7 +1016,7 @@ " flights.loc[:, flights.columns.str.contains(\"TIME\")]\n", " ```\n", "\n", - " (Hint: you can use help even on functions that apply to data frames, eg use `help(flights.columns.str.contains)`)" + " (Hint: you can use help even on functions that apply to data frames, eg use `help(flights.columns.str.contains)`)\n" ] }, { @@ -981,9 +1034,9 @@ "\n", "### Grouping and Aggregating\n", "\n", - "Let's take a look at creating a group using the `.groupby()` function followed by selecting a column and applying a summary statistic via an aggregation. Note that *aggregation*, via `.agg()`, always produces a new index because we have collapsed information down to the group-level (and the new index is made of those levels).\n", + "Let's take a look at creating a group using the `.groupby()` function followed by selecting a column and applying a summary statistic via an aggregation. Note that _aggregation_, via `.agg()`, always produces a new index because we have collapsed information down to the group-level (and the new index is made of those levels).\n", "\n", - "The key point to remember is: use `.agg()` with `.groupby()` when you want your groups to become the new index." + "The key point to remember is: use `.agg()` with `.groupby()` when you want your groups to become the new index.\n" ] }, { @@ -993,7 +1046,7 @@ "metadata": {}, "outputs": [], "source": [ - "(flights.groupby(\"month\")[[\"dep_delay\"]].mean())" + "(flights.group_by(\"month\")[[\"dep_delay\"]].mean())" ] }, { @@ -1003,7 +1056,7 @@ "source": [ "This now represents the mean departure delay by month. Notice that our index has changed! We now have month where we original had an index that was just the row number. The index plays an important role in grouping operations because it keeps track of the groups you have in the rest of your data frame.\n", "\n", - "Often, you might want to do multiple summary operations in one go. The most comprehensive syntax for this is via `.agg()`. We can reproduce what we did above using `.agg()`:" + "Often, you might want to do multiple summary operations in one go. The most comprehensive syntax for this is via `.agg()`. We can reproduce what we did above using `.agg()`:\n" ] }, { @@ -1013,7 +1066,7 @@ "metadata": {}, "outputs": [], "source": [ - "(flights.groupby(\"month\")[[\"dep_delay\"]].agg(\"mean\"))" + "(flights.group_by(\"month\")[[\"dep_delay\"]].agg(\"mean\"))" ] }, { @@ -1023,19 +1076,19 @@ "source": [ "where you pass in whatever aggregation you want. Some common options are in the table below:\n", "\n", - "| Aggregation | Description |\n", - "| ----------- | ----------- |\n", - "| `count()` | Number of items |\n", - "| `first()`, `last()` | \tFirst and last item |\n", - "| `mean()`, `median()` |\tMean and median |\n", - "| `min()`, `max()` |\tMinimum and maximum |\n", - "| `std()`, `var()` |\tStandard deviation and variance |\n", - "| `mad()` |\tMean absolute deviation |\n", - "| `prod()` |\tProduct of all items |\n", - "| `sum()`\t| Sum of all items |\n", - "| `value_counts()` | Counts of unique values |\n", + "| Aggregation | Description |\n", + "| -------------------- | ------------------------------- |\n", + "| `count()` | Number of items |\n", + "| `first()`, `last()` | First and last item |\n", + "| `mean()`, `median()` | Mean and median |\n", + "| `min()`, `max()` | Minimum and maximum |\n", + "| `std()`, `var()` | Standard deviation and variance |\n", + "| `mad()` | Mean absolute deviation |\n", + "| `prod()` | Product of all items |\n", + "| `sum()` | Sum of all items |\n", + "| `value_counts()` | Counts of unique values |\n", "\n", - "For doing multiple aggregations on multiple columns with new names for the output variables, the syntax becomes" + "For doing multiple aggregations on multiple columns with new names for the output variables, the syntax becomes\n" ] }, { @@ -1058,7 +1111,7 @@ "id": "c331e813", "metadata": {}, "source": [ - "Means and counts can get you a surprisingly long way in data science!" + "Means and counts can get you a surprisingly long way in data science!\n" ] }, { @@ -1068,7 +1121,7 @@ "source": [ "### Grouping by multiple variables\n", "\n", - "This is as simple as passing `.groupby()` a list representing multiple columns instead of a string representing a single column." + "This is as simple as passing `.groupby()` a list representing multiple columns instead of a string representing a single column.\n" ] }, { @@ -1078,7 +1131,7 @@ "metadata": {}, "outputs": [], "source": [ - "month_year_delay = flights.groupby([\"month\", \"year\"]).agg(\n", + "month_year_delay = flights.group_by([\"month\", \"year\"]).agg(\n", " mean_delay=(\"dep_delay\", \"mean\"),\n", " count_flights=(\"dep_delay\", \"count\"),\n", ")\n", @@ -1090,45 +1143,7 @@ "id": "22e89b7e", "metadata": {}, "source": [ - "You might have noticed that this time we have a multi-index (that is, an index with more than one column). That's because we asked for something with multiple groups, and the index tracks what's going on within each group: so we need more than one dimension of index to do this efficiently.\n", - "\n", - "If you ever want to go back to an index that is just the position, try `reset_index()`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "33712dc2", - "metadata": {}, - "outputs": [], - "source": [ - "month_year_delay.reset_index()" - ] - }, - { - "cell_type": "markdown", - "id": "96c8416a", - "metadata": {}, - "source": [ - "Perhaps you only want to remove one layer of the index though. This can be achieved by passing the position of the index you'd like to remove: for example, to only change the year index to a column, we would use: " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bcbef8f7", - "metadata": {}, - "outputs": [], - "source": [ - "month_year_delay.reset_index(1)" - ] - }, - { - "cell_type": "markdown", - "id": "8ab554b2", - "metadata": {}, - "source": [ - "Finally, you can do more complicated re-arrangements of the index with an operation called `unstack`, which pivots the chosen index variable to be a column variable instead (introducing a multi column level structure). It's usually best to avoid this." + "You might have noticed that this time we have a multi-index (that is, an index with more than one column). That's because we asked for something with multiple groups, and the index tracks what's going on within each group: so we need more than one dimension of index to do this efficiently.\n" ] }, { @@ -1142,7 +1157,7 @@ "\n", "The key point to remember is: use `.transform()` with `.groupby()` when you want to perform computations on your groups but you want to go back to the original index.\n", "\n", - "Let's say we wanted to express the arrival delay, `\"arr_del\"`, of each flight as a fraction of the worst arrival delay in each month." + "Let's say we wanted to express the arrival delay, `\"arr_del\"`, of each flight as a fraction of the worst arrival delay in each month.\n" ] }, { @@ -1164,7 +1179,7 @@ "id": "3880ece9", "metadata": {}, "source": [ - "Note that the first few entries of `\"max_delay_month\"` are all the same because the month is the same for those entries, but the delay fraction changes with each row." + "Note that the first few entries of `\"max_delay_month\"` are all the same because the month is the same for those entries, but the delay fraction changes with each row.\n" ] }, { @@ -1178,14 +1193,11 @@ "\n", "2. Find the most delayed flight to each destination.\n", "\n", - "3. How do delays vary over the course of the day?" + "3. How do delays vary over the course of the day?\n" ] } ], "metadata": { - "interpreter": { - "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" - }, "jupytext": { "cell_metadata_filter": "-all", "encoding": "# -*- coding: utf-8 -*-", @@ -1193,7 +1205,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "python4ds", "language": "python", "name": "python3" }, diff --git a/data-visualise.ipynb b/data-visualise.ipynb index 0335c48..81e2e28 100644 --- a/data-visualise.ipynb +++ b/data-visualise.ipynb @@ -17,7 +17,7 @@ "\n", "However, we'll get further faster by learning one system and applying it in many places—and the beauty of declarative plotting is that it covers lots of standard charts simply and well. **letsplot** implements the so-called **grammar of graphics**, a coherent declarative system for describing and building graphs.\n", "\n", - "We will start by creating a simple scatterplot and use that to introduce aesthetic mappings and geometric objects—the fundamental building blocks of **letsplot**. We will then walk you through visualising distributions of single variables as well as visualising relationships between two or more variables. We’ll finish off with saving your plots and troubleshooting tips. " + "We will start by creating a simple scatterplot and use that to introduce aesthetic mappings and geometric objects—the fundamental building blocks of **letsplot**. We will then walk you through visualising distributions of single variables as well as visualising relationships between two or more variables. We’ll finish off with saving your plots and troubleshooting tips.\n" ] }, { @@ -27,7 +27,7 @@ "source": [ "### Prerequisites\n", "\n", - "You will need to install the **letsplot** package for this chapter. To do this, open up the command line of your computer, type in `uv add lets-plot`, and hit enter." + "You will need to install the **letsplot** package for this chapter. To do this, open up the command line of your computer, type in `uv add lets-plot`, and hit enter.\n" ] }, { @@ -39,7 +39,7 @@ "The command line can be opened within Visual Studio Code and Codespaces by going to View -> Terminal.\n", ":::\n", "\n", - "Note that you only need to install a package once in each Python environment." + "Note that you only need to install a package once in each Python environment.\n" ] }, { @@ -47,9 +47,9 @@ "id": "e0ad70c8", "metadata": {}, "source": [ - "We'll also need to have the **pandas** package installed—this package, which we'll be seeing a lot of, is for data. You can similarly install it by running `uv add pandas` on the command line.\n", + "We'll also need to have the **polars** package installed—this package, which we'll be seeing a lot of, is for data. You can similarly install it by running `uv add polars` on the command line.\n", "\n", - "Finally, we'll also need some data (you can't science without data). We'll be using the Palmer penguins dataset. Unusually, this can also be installed as a package—normally you would load data from a file, but these data are so popular for tutorials they've found their way into an installable package. Run `uv add palmerpenguins` to get these data." + "Finally, we'll also need some data (you can't science without data). We'll be using the Palmer penguins dataset. Unusually, this can also be installed as a package—normally you would load data from a file, but these data are so popular for tutorials they've found their way into an installable package. Run `uv add palmerpenguins` to get these data.\n" ] }, { @@ -57,18 +57,60 @@ "id": "8852373a", "metadata": {}, "source": [ - "Our next task is to load these into our Python session, either in a Python notebook cell within a Jupyter Notebook, by writing it in a script that we then send to the interactive window, or by typing it directly into the interactive window and hitting shift and enter. Here's the code:" + "Our next task is to load these into our Python session, either in a Python notebook cell within a Marimo or Jupyter Notebook, by writing it in a script that we then send to the interactive window, or by typing it directly into the interactive window and hitting shift and enter. Here's the code:\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "a86fb211", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "from lets_plot import *\n", "from palmerpenguins import load_penguins\n", + "import polars as pl\n", "\n", "LetsPlot.setup_html()" ] @@ -78,7 +120,7 @@ "id": "4443f4dd", "metadata": {}, "source": [ - "These lines import parts of the **pandas** and **palmerpenguins** packages, then import all (`*`) of the functions of the **letsplot** package. The final line allows charts to display in HTML." + "These lines import parts of the **palmerpenguins** package, then import all (`*`) of the functions of the **letsplot** package. The final line allows charts to display in HTML.\n" ] }, { @@ -88,7 +130,7 @@ "source": [ "## First Steps\n", "\n", - "Do penguins with longer flippers weigh more or less than penguins with shorter flippers? You probably already have an answer, but try to make your answer precise. What does the relationship between flipper length and body mass look like? Is it positive? Negative? Linear? Nonlinear? Does the relationship vary by the species of the penguin? How about by the island where the penguin lives? Let’s create visualisations that we can use to answer these questions." + "Do penguins with longer flippers weigh more or less than penguins with shorter flippers? You probably already have an answer, but try to make your answer precise. What does the relationship between flipper length and body mass look like? Is it positive? Negative? Linear? Nonlinear? Does the relationship vary by the species of the penguin? How about by the island where the penguin lives? Let’s create visualisations that we can use to answer these questions.\n" ] }, { @@ -102,21 +144,21 @@ "\n", "To make the discussion easier, let's define some terms:\n", "\n", - "- A **variable** is a quantity, quality, or property that you can measure.\n", + "- A **variable** is a quantity, quality, or property that you can measure.\n", "\n", - "- A **value** is the state of a variable when you measure it.\n", - " The value of a variable may change from measurement to measurement.\n", + "- A **value** is the state of a variable when you measure it.\n", + " The value of a variable may change from measurement to measurement.\n", "\n", - "- An **observation** is a set of measurements made under similar conditions (you usually make all of the measurements in an observation at the same time and on the same object).\n", - " An observation will contain several values, each associated with a different variable.\n", - " We'll sometimes refer to an observation as a data point.\n", + "- An **observation** is a set of measurements made under similar conditions (you usually make all of the measurements in an observation at the same time and on the same object).\n", + " An observation will contain several values, each associated with a different variable.\n", + " We'll sometimes refer to an observation as a data point.\n", "\n", - "- **Tabular data** is a set of values, each associated with a variable and an observation.\n", - " Tabular data is *tidy* if each value is placed in its own \"cell\", each variable in its own column, and each observation in its own row.\n", + "- **Tabular data** is a set of values, each associated with a variable and an observation.\n", + " Tabular data is _tidy_ if each value is placed in its own \"cell\", each variable in its own column, and each observation in its own row.\n", "\n", "In this context, a variable refers to an attribute of all the penguins, and an observation refers to all the attributes of a single penguin.\n", "\n", - "Type the name of the data frame in the interactive window and Python will print a preview of its contents." + "Type the name of the data frame in the interactive window and Python will print a preview of its contents.\n" ] }, { @@ -126,7 +168,7 @@ "metadata": {}, "outputs": [], "source": [ - "penguins = load_penguins()\n", + "penguins = pl.from_pandas(load_penguins())\n", "penguins" ] }, @@ -135,7 +177,7 @@ "id": "cc310b4f", "metadata": {}, "source": [ - "For an alternative view, where you can see the first few observations of each variable, use `penguins.head()`." + "For an alternative view, where you can see the first few observations of each variable, use `penguins.head()`.\n" ] }, { @@ -171,7 +213,7 @@ "source": [ "### Ultimate Goal\n", "\n", - "Our ultimate goal in this chapter is to recreate the following visualisation displaying the relationship between flipper lengths and body masses of these penguins, taking into consideration the species of the penguin." + "Our ultimate goal in this chapter is to recreate the following visualisation displaying the relationship between flipper lengths and body masses of these penguins, taking into consideration the species of the penguin.\n" ] }, { @@ -241,7 +283,7 @@ "For example, bar charts use bar geoms (`geom_bar()`), line charts use line geoms (`geom_line()`), boxplots use boxplot geoms (`geom_boxplot()`), scatterplots use point geoms (`geom_point()`), and so on.\n", "\n", "The function `geom_point()` adds a layer of points to your plot, which creates a scatterplot.\n", - "**letsplot** comes with many geom functions that each adds a different type of layer to a plot." + "**letsplot** comes with many geom functions that each adds a different type of layer to a plot.\n" ] }, { @@ -266,7 +308,7 @@ "It doesn't yet match our \"ultimate goal\" plot, but using this plot we can start answering the question that motivated our exploration: \"What does the relationship between flipper length and body mass look like?\" The relationship appears to be positive (as flipper length increases, so does body mass), fairly linear (the points are clustered around a line instead of a curve), and moderately strong (there isn't too much scatter around such a line).\n", "Penguins with longer flippers are generally larger in terms of their body mass.\n", "\n", - "It's a good point to flag that although we have plotted everything in the `penguins` data frame, there were a couple of rows with undefined values—and of course these cannot be plotted." + "It's a good point to flag that although we have plotted everything in the `penguins` data frame, there were a couple of rows with undefined values—and of course these cannot be plotted.\n" ] }, { @@ -285,7 +327,7 @@ "If you guessed \"in the aesthetic mapping, inside of `aes()`\", you're already getting the hang of creating data visualisations with **letsplot**!\n", "And if not, don't worry.\n", "\n", - "Throughout the book you will make many more plots and have many more opportunities to check your intuition as you make them." + "Throughout the book you will make many more plots and have many more opportunities to check your intuition as you make them.\n" ] }, { @@ -319,7 +361,7 @@ "\n", "Since this is a new geometric object representing our data, we will add a new geom as a layer on top of our point geom: `geom_smooth()`.\n", "\n", - "And we will specify that we want to draw the line of best fit based on a `l`inear `m`odel with `method = \"lm\"`." + "And we will specify that we want to draw the line of best fit based on a `l`inear `m`odel with `method = \"lm\"`.\n" ] }, { @@ -346,9 +388,9 @@ "source": [ "We have successfully added lines, but this plot doesn't look like the plot from earlier as that only had one line for the entire dataset as opposed to separate lines for each of the penguin species.\n", "\n", - "When aesthetic mappings are defined in `ggplot()`, at the *global* level, they're passed down to each of the subsequent geom layers of the plot.\n", + "When aesthetic mappings are defined in `ggplot()`, at the _global_ level, they're passed down to each of the subsequent geom layers of the plot.\n", "\n", - "However, each geom function in **letplot** can also take a `mapping` argument, which allows for aesthetic mappings at the *local* level that are added to those inherited from the global level.\n", + "However, each geom function in **letplot** can also take a `mapping` argument, which allows for aesthetic mappings at the _local_ level that are added to those inherited from the global level.\n", "\n", "Since we want points to be colored based on species but don't want the lines to be separated out for them, we should specify `color = species` for `geom_point()` only: therefore we take it out of the global `aes()` and just add it to `geom_point()`.\n" ] @@ -376,7 +418,7 @@ "\n", "We still need to use different shapes for each species of penguins and improve labels.\n", "\n", - "It's generally not a good idea to represent information using only colors on a plot, as people perceive colors differently due to color blindness or other color vision differences. Therefore, in addition to color, we can also map `species` to the `shape` aesthetic." + "It's generally not a good idea to represent information using only colors on a plot, as people perceive colors differently due to color blindness or other color vision differences. Therefore, in addition to color, we can also map `species` to the `shape` aesthetic.\n" ] }, { @@ -400,7 +442,7 @@ "source": [ "Note that the legend is automatically updated to reflect the different shapes of the points as well.\n", "\n", - "And finally, we can improve the labels of our plot using the `labs()` function in a new layer. Some of the arguments to `labs()` might be self explanatory: `title` adds a title and `subtitle` adds a subtitle to the plot. Other arguments match the aesthetic mappings, `x` is the x-axis label, `y` is the y-axis label, and `color` and `shape` define the label for the legend." + "And finally, we can improve the labels of our plot using the `labs()` function in a new layer. Some of the arguments to `labs()` might be self explanatory: `title` adds a title and `subtitle` adds a subtitle to the plot. Other arguments match the aesthetic mappings, `x` is the x-axis label, `y` is the y-axis label, and `color` and `shape` define the label for the legend.\n" ] }, { @@ -430,7 +472,7 @@ "id": "cdc33b33", "metadata": {}, "source": [ - "We finally have a plot that perfectly matches our \"ultimate goal\"!" + "We finally have a plot that perfectly matches our \"ultimate goal\"!\n" ] }, { @@ -456,7 +498,7 @@ "5. Why does the following give an error and how would you fix it?\n", "\n", " ```python\n", - " (ggplot(data = penguins) + \n", + " (ggplot(data = penguins) +\n", " geom_point())\n", " ```\n", "\n", @@ -464,7 +506,7 @@ "\n", "7. Recreate the following visualisation.\n", " What aesthetic should `bill_depth_mm` be mapped to?\n", - " And should it be mapped at the global level or at the geom level?" + " And should it be mapped at the global level or at the geom level?\n" ] }, { @@ -490,7 +532,6 @@ "id": "986fdc29", "metadata": {}, "source": [ - "\n", "8. Run this code in your head and predict what the output will look like.\n", " Then, run the code in Python and check your predictions.\n", "\n", @@ -518,6 +559,7 @@ " geom_smooth()\n", " )\n", " ```\n", + "\n", " ```python\n", " (ggplot() +\n", " geom_point(\n", @@ -529,7 +571,7 @@ " mapping = aes(x = \"flipper_length_mm\", y = \"body_mass_g\")\n", " )\n", " )\n", - " ```" + " ```\n" ] }, { @@ -549,7 +591,7 @@ " mapping = aes(x = \"flipper_length_mm\", y = \"body_mass_g\")\n", ") +\n", " geom_point())\n", - "```" + "```\n" ] }, { @@ -565,10 +607,10 @@ "\n", "```python\n", "(\n", - " ggplot(penguins, aes(x = \"flipper_length_mm\", y = \"body_mass_g\")) + \n", + " ggplot(penguins, aes(x = \"flipper_length_mm\", y = \"body_mass_g\")) +\n", " geom_point()\n", ")\n", - "```" + "```\n" ] }, { @@ -602,9 +644,9 @@ "id": "699f42eb", "metadata": {}, "source": [ - "You may have seen earlier that the *data type* of the `\"species\"` column is string. Ideally, we want it to be categorical, so that there's no confusion about the fact that we're dealing with a finite number of mutually exclusive groups here. Another advantage is that it allows plotting tools to realise what kind of data it is working with.\n", + "You may have seen earlier that the _data type_ of the `\"species\"` column is string. Ideally, we want it to be categorical, so that there's no confusion about the fact that we're dealing with a finite number of mutually exclusive groups here. Another advantage is that it allows plotting tools to realise what kind of data it is working with.\n", "\n", - "We can transform the variable to a categorical variable using **pandas** like so:" + "We can transform the variable to a categorical variable using **polars** like so:\n" ] }, { @@ -614,7 +656,7 @@ "metadata": {}, "outputs": [], "source": [ - "penguins[\"species\"] = penguins[\"species\"].astype(\"category\")\n", + "penguins = penguins.cast({\"species\": pl.Categorical})\n", "penguins.head()" ] }, @@ -623,7 +665,7 @@ "id": "06d834a5", "metadata": {}, "source": [ - "You will learn more about categorical variables later in the book." + "You will learn more about categorical variables later in the book.\n" ] }, { @@ -631,12 +673,11 @@ "id": "f9ca3124", "metadata": {}, "source": [ - "\n", "### A numerical variable\n", "\n", "A variable is **numerical** (or quantitative) if it can take on a wide range of numerical values, and it is sensible to add, subtract, or take averages with those values. Numerical variables can be continuous or discrete.\n", "\n", - "One commonly used visualisation for distributions of continuous variables is a histogram." + "One commonly used visualisation for distributions of continuous variables is a histogram.\n" ] }, { @@ -661,7 +702,7 @@ "You should always explore a variety of binwidths when working with histograms, as different binwidths can reveal different patterns.\n", "In the plots below a binwidth of 20 is too narrow, resulting in too many bars, making it difficult to determine the shape of the distribution.\n", "Similarly, a binwidth of 2,000 is too high, resulting in all data being binned into only three bars, and also making it difficult to determine the shape of the distribution.\n", - "A binwidth of 200 provides a sensible balance, but you should always look at your data a few different ways, especially with histograms as they can be misleading." + "A binwidth of 200 provides a sensible balance, but you should always look at your data a few different ways, especially with histograms as they can be misleading.\n" ] }, { @@ -710,7 +751,7 @@ " geom_bar(fill = \"red\"))\n", " ```\n", "\n", - "3. What does the `bins` argument in `geom_histogram()` do?" + "3. What does the `bins` argument in `geom_histogram()` do?\n" ] }, { @@ -722,7 +763,7 @@ "\n", "To visualise a relationship we need to have at least two variables mapped to aesthetics of a plot—though you should remember that correlation is not causation, and causation is not correlation!\n", "\n", - "In the following sections you will learn about commonly used plots for visualising relationships between two or more variables and the geoms used for creating them." + "In the following sections you will learn about commonly used plots for visualising relationships between two or more variables and the geoms used for creating them.\n" ] }, { @@ -738,17 +779,16 @@ "\n", "It is also useful for identifying potential outliers. Each boxplot consists of:\n", "\n", - "- A box that indicates the range of the middle half of the data, a distance known as the interquartile range (IQR), stretching from the 25th percentile of the distribution to the 75th percentile.\n", - " In the middle of the box is a line that displays the median, i.e. 50th percentile, of the distribution.\n", - " These three lines give you a sense of the spread of the distribution and whether or not the distribution is symmetric about the median or skewed to one side.\n", + "- A box that indicates the range of the middle half of the data, a distance known as the interquartile range (IQR), stretching from the 25th percentile of the distribution to the 75th percentile.\n", + " In the middle of the box is a line that displays the median, i.e. 50th percentile, of the distribution.\n", + " These three lines give you a sense of the spread of the distribution and whether or not the distribution is symmetric about the median or skewed to one side.\n", "\n", - "- Visual points that display observations that fall more than 1.5 times the IQR from either edge of the box.\n", - " These outlying points are unusual so are plotted individually.\n", + "- Visual points that display observations that fall more than 1.5 times the IQR from either edge of the box.\n", + " These outlying points are unusual so are plotted individually.\n", "\n", - "- A line (or whisker) that extends from each end of the box and goes to the farthest non-outlier point in the distribution.\n", + "- A line (or whisker) that extends from each end of the box and goes to the farthest non-outlier point in the distribution.\n", "\n", - "\n", - "Let's take a look at the distribution of body mass by species using `geom_boxplot()`:" + "Let's take a look at the distribution of body mass by species using `geom_boxplot()`:\n" ] }, { @@ -766,7 +806,7 @@ "id": "97b24caa", "metadata": {}, "source": [ - "Alternatively, we can make probability density plots with `geom_density()`." + "Alternatively, we can make probability density plots with `geom_density()`.\n" ] }, { @@ -788,7 +828,7 @@ "\n", "Additionally, we can map `species` to both `color` and `fill` aesthetics and use the `alpha` aesthetic to add transparency to the filled density curves.\n", "This aesthetic takes values between 0 (completely transparent) and 1 (completely opaque).\n", - "In the following plot it's *set* to 0.5." + "In the following plot it's _set_ to 0.5.\n" ] }, { @@ -811,8 +851,8 @@ "source": [ "Note the terminology we have used here:\n", "\n", - "- We *map* variables to aesthetics if we want the visual attribute represented by that aesthetic to vary based on the values of that variable.\n", - "- Otherwise, we *set* the value of an aesthetic.\n" + "- We _map_ variables to aesthetics if we want the visual attribute represented by that aesthetic to vary based on the values of that variable.\n", + "- Otherwise, we _set_ the value of an aesthetic.\n" ] }, { @@ -829,7 +869,7 @@ "The first plot shows the frequencies of each species of penguins on each island.\n", "The plot of frequencies show that there are equal numbers of Adelies on each island.\n", "\n", - "But we don't have a good sense of the percentage balance within each island." + "But we don't have a good sense of the percentage balance within each island.\n" ] }, { @@ -867,7 +907,7 @@ "id": "cc83c3db", "metadata": {}, "source": [ - "In creating these bar charts, we map the variable that will be separated into bars to the `x` aesthetic, and the variable that will change the colors inside the bars to the `fill` aesthetic." + "In creating these bar charts, we map the variable that will be separated into bars to the `x` aesthetic, and the variable that will change the colors inside the bars to the `fill` aesthetic.\n" ] }, { @@ -927,7 +967,7 @@ "\n", "To facet your plot by a single variable, use `facet_wrap()`.\n", "\n", - "The first argument of `facet_wrap()` tells the function what variable to have in successive charts. The variable that you pass to `facet_wrap()` should be categorical." + "The first argument of `facet_wrap()` tells the function what variable to have in successive charts. The variable that you pass to `facet_wrap()` should be categorical.\n" ] }, { @@ -949,7 +989,7 @@ "id": "ee5a3eed", "metadata": {}, "source": [ - "You will learn about many other geoms for visualising distributions of variables and relationships between them in later chapters." + "You will learn about many other geoms for visualising distributions of variables and relationships between them in later chapters.\n" ] }, { @@ -971,7 +1011,7 @@ " ggplot(\n", " data = penguins,\n", " mapping = aes(\n", - " x = \"bill_length_mm\", y = \"bill_depth_mm\", \n", + " x = \"bill_length_mm\", y = \"bill_depth_mm\",\n", " color = \"species\", shape = \"species\"\n", " )\n", " ) +\n", @@ -1023,9 +1063,9 @@ "source": [ "This saved the figure to disk at the location shown—by default it's in a subdirectory called \"lets-plot-images\".\n", "\n", - "We used the file format \"svg\". There are lots of output options to choose from to save your file to. Remember that, for graphics, *vector formats* are generally better than *raster formats*. In practice, this means saving plots in svg or pdf formats over jpg or png file formats. The svg format works in a lot of contexts (including Microsoft Word) and is a good default. To choose between formats, just supply the file extension and the file type will change automatically, eg \"chart.svg\" for svg or \"chart.png\" for png. You can also save figures in HTML format.\n", + "We used the file format \"svg\". There are lots of output options to choose from to save your file to. Remember that, for graphics, _vector formats_ are generally better than _raster formats_. In practice, this means saving plots in svg or pdf formats over jpg or png file formats. The svg format works in a lot of contexts (including Microsoft Word) and is a good default. To choose between formats, just supply the file extension and the file type will change automatically, eg \"chart.svg\" for svg or \"chart.png\" for png. You can also save figures in HTML format.\n", "\n", - "If you're using a raster format then you'll need to specify how big the figure is via the *scale* keyword argument." + "If you're using a raster format then you'll need to specify how big the figure is via the _scale_ keyword argument.\n" ] }, { @@ -1051,7 +1091,7 @@ "source": [ "### Exercises\n", "\n", - "1. Save the figure above as a PNG. Try varying the scale." + "1. Save the figure above as a PNG. Try varying the scale.\n" ] }, { @@ -1066,13 +1106,12 @@ "We have all been writing Python code for years, but every day we still write code that doesn't work on the first try!\n", "\n", "Start by carefully comparing the code that you're running to the code in the book: A misplaced character can make all the difference!\n", - "Make sure that every `(` is matched with a `)` and every `\"` is paired with another `\"`. In Visual Studio Code, you can get extensions that colour match brackets so you can easily see if you closed them or not.\n", + "Make sure that every `(` is matched with a `)` and every `\"` is paired with another `\"`. In Visual Studio Code, you can get extensions that colour match brackets so you can easily see if you closed them or not.\n", "\n", "Sometimes you'll run the code and nothing happens.\n", "\n", "For those coming from the R statistical programming language, you may be concerned about getting your `+` in the wrong place. Have no fear, however, as in the syntax for **letsplot** the `+` can go at the start or the end of the line.\n", "\n", - "\n", "If you're still stuck, try the help.\n", "You can get help about any Python function by running `help(function_name)` in the interactive window.\n", "Don't worry if the help doesn't seem that helpful - instead skip down to the examples and look for code that matches what you're trying to do.\n", @@ -1095,7 +1134,7 @@ "We'll use visualisations again and again throughout this book, introducing new techniques as we need them as well as do a deeper dive into creating visualisations with **letsplot** in subsequent chapters.\n", "\n", "With the basics of visualisation under your belt, in the next chapter we're going to switch gears a little and give you some practical workflow advice.\n", - "We intersperse workflow advice with data science tools throughout this part of the book because it'll help you stay organised as you write more Python code." + "We intersperse workflow advice with data science tools throughout this part of the book because it'll help you stay organised as you write more Python code.\n" ] } ], @@ -1107,7 +1146,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": ".venv", + "display_name": "python4ds", "language": "python", "name": "python3" }, diff --git a/uv.lock b/uv.lock index 8316606..5342f08 100644 --- a/uv.lock +++ b/uv.lock @@ -1725,7 +1725,7 @@ wheels = [ [[package]] name = "python4ds" -version = "1.0.4" +version = "0.0.1" source = { virtual = "." } dependencies = [ { name = "beautifulsoup4" }, From 97c56c3668e1e8c9a1d0332d3e3056c9bd2c3c35 Mon Sep 17 00:00:00 2001 From: Uchenna Ugoh Date: Wed, 22 Apr 2026 14:57:22 +0100 Subject: [PATCH 10/34] WIP: Data Transformation --- data-transform.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-transform.ipynb b/data-transform.ipynb index eb7c6d0..0465933 100644 --- a/data-transform.ipynb +++ b/data-transform.ipynb @@ -528,7 +528,7 @@ "outputs": [], "source": [ "(\n", - " flights.filter(pl.col(\"dep_delay\") <= 10) & (pl.col(\"dep_delay\") >= -10)\n", + " flights.filter((pl.col(\"dep_delay\") <= 10) & (pl.col(\"dep_delay\") >= -10))\n", " .sort(\"arr_delay\", descending=True)\n", " .head(3)\n", ")" From e7456153a9e52ad92b8bd05abd16f43545ad586e Mon Sep 17 00:00:00 2001 From: Nwabueze Ugoh Date: Wed, 22 Apr 2026 15:11:15 +0100 Subject: [PATCH 11/34] Replace pandas with polars in rectangling.ipynb and webscraping-and-apis.ipynb, updating relevant code and documentation accordingly. --- rectangling.ipynb | 34 ++-- visualise.quarto_ipynb_1 | 136 ++++++++++++++++ webscraping-and-apis.ipynb | 130 +++++++-------- workflow-help.quarto_ipynb_1 | 115 ++++++++++++++ ...w-packages-and-environments.quarto_ipynb_1 | 149 ++++++++++++++++++ 5 files changed, 483 insertions(+), 81 deletions(-) create mode 100644 visualise.quarto_ipynb_1 create mode 100644 workflow-help.quarto_ipynb_1 create mode 100644 workflow-packages-and-environments.quarto_ipynb_1 diff --git a/rectangling.ipynb b/rectangling.ipynb index f0bcf47..09d3402 100644 --- a/rectangling.ipynb +++ b/rectangling.ipynb @@ -41,7 +41,7 @@ "source": [ "### Prerequisites\n", "\n", - "This chapter will use the **pandas** data analysis package." + "This chapter will use the **polars** data analysis package." ] }, { @@ -292,9 +292,9 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import polars as pl\n", "\n", - "pd.DataFrame(data=list_of_lists, columns=[\"a\", \"b\", \"c\"])" + "df = pl.DataFrame(data=list_of_lists, schema=[\"a\", \"b\", \"c\"])" ] }, { @@ -312,12 +312,13 @@ "metadata": {}, "outputs": [], "source": [ - "df = pd.DataFrame(\n", + "df = pl.DataFrame(\n", " {\n", - " \"alpha\": [[0, 1, 2], \"foo\", [], [3, 4]],\n", - " \"beta\": 1,\n", - " \"gamma\": [[\"a\", \"b\", \"c\"], pd.NA, [], [\"d\", \"e\"]],\n", - " }\n", + " \"alpha\": [[\"0,1,2\"], \"foo\", [], [\"3,4\"]],\n", + " \"beta\": [1, 1, 1, 1],\n", + " \"gamma\": [[\"a\", \"b\", \"c\"], None, [], [\"d\", \"e\"]],\n", + " },\n", + " strict=False,\n", ")\n", "df" ] @@ -540,7 +541,7 @@ "source": [ "### From JSON data to Data Frame\n", "\n", - "**pandas** has lots of options for turning JSON or dictionary data into a data frame. You do need to think a little bit about the structure of the data underneath though:\n" + "**polars** has lots of options for turning JSON or dictionary data into a data frame. You do need to think a little bit about the structure of the data underneath though:\n" ] }, { @@ -550,9 +551,9 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import polars as pl\n", "\n", - "pd.DataFrame(result[\"toppings\"], columns=[\"id\", \"type\"])" + "df = pl.DataFrame(result[\"toppings\"], schema=[\"id\", \"type\"])" ] }, { @@ -560,7 +561,7 @@ "id": "a1346020", "metadata": {}, "source": [ - "The web-scraped data we downloaded earlier had a more complicated structure, but **pandas** has a `json_normalize()` function that can cope with this. For example, with the following data, there are many missing entries but `json_normalize()` can still parse it into a Data Frame." + "The web-scraped data we downloaded earlier had a more complicated structure, but **polars** has a `json_normalize()` function that can cope with this. For example, with the following data, there are many missing entries but `json_normalize()` can still parse it into a Data Frame." ] }, { @@ -575,7 +576,8 @@ " {\"name\": {\"given\": \"Mark\", \"family\": \"Regner\"}},\n", " {\"id\": 2, \"name\": \"Faye Raker\"},\n", "]\n", - "pd.json_normalize(data)" + "pl.json_normalize(data)\n", + "\n" ] }, { @@ -583,7 +585,7 @@ "id": "7eaf00e1", "metadata": {}, "source": [ - "And we can control the level that properties like 'name' are split out to as well (you can check out more options over at the [**pandas** documentation](https://pandas.pydata.org/docs/reference/api/pandas.json_normalize.html))" + "And we can control the level that properties like 'name' are split out to as well (you can check out more options over at the [**polars** documentation](https://docs.pola.rs/api/python/stable/reference/api/polars.json_normalize.html))" ] }, { @@ -593,7 +595,7 @@ "metadata": {}, "outputs": [], "source": [ - "pd.json_normalize(data, max_level=0)" + "pl.json_normalize(data, max_level=0)" ] }, { @@ -601,7 +603,7 @@ "id": "78d637e5", "metadata": {}, "source": [ - "As well as the JSON normalise function, **pandas** has a `from_dict()` method to work with simpler dictionary objects." + "As well as the JSON normalise function, **polars** has a `from_dict()` method to work with simpler dictionary objects." ] } ], diff --git a/visualise.quarto_ipynb_1 b/visualise.quarto_ipynb_1 new file mode 100644 index 0000000..eb958d5 --- /dev/null +++ b/visualise.quarto_ipynb_1 @@ -0,0 +1,136 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Visualisation {#sec-visualise}\n", + "\n", + "After reading the first part of the book, you understand the basics of the most important tools for doing data science. Now it’s time to start diving into the details. In this part of the book, you’ll learn about visualising data in further depth (in @sec-vis-layers), and get further stuck into the details of the different kinds of data visualisation (in @sec-exploratory-data-analysis and @sec-communicate-plots). In this short chapter, we discuss the different ways to create visualisations, and the different purposes of visualisations.\n", + "\n", + "## Philosophies of data visualisation\n", + "\n", + "There are broadly two categories of approach to using code to create data visualisations: *imperative* (build what you want from individual elements) and *declarative* (say what you want from a list of pre-existing options). Choosing which to use involves a trade-off: imperative libraries offer you flexibility but at the cost of some verbosity; declarative libraries offer you a quick way to plot your data, but only if it’s in the right format to begin with, and customisation to special chart types is more difficult.\n", + "\n", + "Python has many excellent plotting packages, including perhaps the most powerful imperative plotting package around, **matplotlib**, and an amazing declarative library that we already saw, **lets-plot**. These two libraries will get you a long way, and each could be worthy of an entire book themselves. Fortunately for us, though, we can do 95% of what we need with a small number of commands from one or the other of them. In general, to keep this book as light as possible, we've opted to use **lets-plot** wherever possible—and @sec-vis-layers is going to take you on a more in-depth tour of how to use it yourself.\n", + "\n", + "## Purposes of data visualisation\n", + "\n", + "Data visualisation has all kinds of different purposes. It can be useful to bear in mind three broad categories of visualisation that are out there:\n", + "\n", + "- exploratory\n", + "- scientific\n", + "- narrative\n", + "\n", + "Let's look at each in a bit more detail.\n", + "\n", + "### Exploratory Data Viz\n", + "\n", + "The first of the three kinds is *exploratory data visualisation*, and it's the kind that you do when you're looking and data and trying to understand it. Just plotting the data is a really good strategy for getting a feel for any issues there might be. This is perhaps most famously demonstrated by Anscombe's quartet: four different datasets with the same mean, standard deviation, and correlation but very different data distributions." + ], + "id": "2d4a4a09" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "#| echo: false\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib_inline.backend_inline\n", + "\n", + "# Plot settings\n", + "plt.style.use(\"https://github.com/aeturrell/python4DS/raw/main/plot_style.txt\")\n", + "matplotlib_inline.backend_inline.set_matplotlib_formats(\"svg\")\n", + "\n", + "# Set max rows displayed for readability\n", + "pd.set_option(\"display.max_rows\", 6)\n", + "\n", + "x = [10, 8, 13, 9, 11, 14, 6, 4, 12, 7, 5]\n", + "y1 = [8.04, 6.95, 7.58, 8.81, 8.33, 9.96, 7.24, 4.26, 10.84, 4.82, 5.68]\n", + "y2 = [9.14, 8.14, 8.74, 8.77, 9.26, 8.10, 6.13, 3.10, 9.13, 7.26, 4.74]\n", + "y3 = [7.46, 6.77, 12.74, 7.11, 7.81, 8.84, 6.08, 5.39, 8.15, 6.42, 5.73]\n", + "x4 = [8, 8, 8, 8, 8, 8, 8, 19, 8, 8, 8]\n", + "y4 = [6.58, 5.76, 7.71, 8.84, 8.47, 7.04, 5.25, 12.50, 5.56, 7.91, 6.89]\n", + "\n", + "datasets = {\"I\": (x, y1), \"II\": (x, y2), \"III\": (x, y3), \"IV\": (x4, y4)}\n", + "\n", + "fig, axs = plt.subplots(\n", + " 2,\n", + " 2,\n", + " sharex=True,\n", + " sharey=True,\n", + " figsize=(10, 6),\n", + " gridspec_kw={\"wspace\": 0.08, \"hspace\": 0.08},\n", + ")\n", + "axs[0, 0].set(xlim=(0, 20), ylim=(2, 14))\n", + "axs[0, 0].set(xticks=(0, 10, 20), yticks=(4, 8, 12))\n", + "\n", + "for ax, (label, (x, y)) in zip(axs.flat, datasets.items()):\n", + " ax.text(0.1, 0.9, label, fontsize=20, transform=ax.transAxes, va=\"top\")\n", + " ax.tick_params(direction=\"in\", top=True, right=True)\n", + " ax.plot(x, y, \"o\")\n", + "\n", + " # linear regression\n", + " p1, p0 = np.polyfit(x, y, deg=1) # slope, intercept\n", + " ax.axline(xy1=(0, p0), slope=p1, color=\"r\", lw=2)\n", + "\n", + " # add text box for the statistics\n", + " stats = (\n", + " f\"$\\\\mu$ = {np.mean(y):.2f}\\n\"\n", + " f\"$\\\\sigma$ = {np.std(y):.2f}\\n\"\n", + " f\"$r$ = {np.corrcoef(x, y)[0][1]:.2f}\"\n", + " )\n", + " bbox = dict(boxstyle=\"round\", fc=\"blanchedalmond\", ec=\"orange\", alpha=0.5)\n", + " ax.text(\n", + " 0.95,\n", + " 0.07,\n", + " stats,\n", + " fontsize=9,\n", + " bbox=bbox,\n", + " transform=ax.transAxes,\n", + " horizontalalignment=\"right\",\n", + " )\n", + "\n", + "plt.suptitle(\"Anscombe's Quartet\")\n", + "plt.show()" + ], + "id": "c38c38fa", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Exploratory visualisation is usually quick and dirty, and flexible too. Some exploratory data viz can be automated, and there's a whole host of packages to help with this, including [**skimpy**](https://aeturrell.github.io/skimpy/).\n", + "\n", + "Beyond you and perhaps your co-authors/collaborators, however, not many other people should be seeing your exploratory visualisation! They will typically be worked up quickly, be numerous, and be throw-away. We'll look more at this in @sec-exploratory-data-analysis.\n", + "\n", + "### Scientific Data Viz\n", + "\n", + "The second kind, scientific data visualisation, is the prime cut of your exploratory visualisation. It's the kind of plot you might include in a more technical paper, the picture that says a thousand words. I often think of the first image of a black hole @akiyama2019first as a prime example of this. You can get away with having a high density of information in a scientific plot and, in short format journals, you may need to. The journal Physical Review Letters, which has an 8 page limit, has a classic of this genre in more or less every issue. Ensuring that important values can be accurately read from the plot is especially important in these kinds of charts. But they can also be the kind of plot that presents the killer results in a study; they might not be exciting to people who don't look at charts for a living, but they might be exciting and, just as importantly, understandable by your peers.\n", + "\n", + "This type of visualisation is especially popular in the big science journals like *Nature* and *Science*, where space is at a premium. We won't cover this type of plot in this book, because it tends to be very bespoke.\n", + "\n", + "### Narrative Data Viz\n", + "\n", + "The third and final kind is narrative data visualisation. This is the one that requires the most thought in the step where you go from the first view to the end product. It's a visualisation that doesn't just show a picture, but gives an insight. These are the kind of visualisations that you might see in the *Financial Times*, *The Economist*, or on the *BBC News* website. They come with aids that help the viewer focus on the aspects that the creator wanted them to (you can think of these aids or focuses as doing for visualisation what bold font does for text). They're well worth using in your work, especially if you're trying to communicate a particular narrative, and especially if the people you're communicating with don't have deep knowledge of the topic. You might use them in a paper that you hope will have a wide readership, in a blog post summarising your work, or in a report intended for a policymaker.\n", + "\n", + "You can find more information on the topic of communicating via data visualisations in the @sec-communicate-plots chapter." + ], + "id": "88d12184" + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "language": "python", + "display_name": "Python 3 (ipykernel)", + "path": "/Users/omagic/Documents/GitHub/python4DSpolars/.venv/share/jupyter/kernels/python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/webscraping-and-apis.ipynb b/webscraping-and-apis.ipynb index 9366eb6..6e8daf7 100644 --- a/webscraping-and-apis.ipynb +++ b/webscraping-and-apis.ipynb @@ -67,7 +67,7 @@ "source": [ "### Prerequisites\n", "\n", - "You will need to install the **pandas** package for this chapter. We'll use **seaborn** too, which you should already have installed. You will also need to install the **beautifulsoup**, **pandas-datareader**, and **wbgapi** packages in your terminal using `uv add beautifulsoup4`, `uv add pandas-datareader`, and `uv add wbgapi` respectively. We'll also use two built-in packages, **textwrap** and **requests**.\n", + "You will need to install the **pandas** and **polars** package for this chapter. We'll use **seaborn** too, which you should already have installed. You will also need to install the **beautifulsoup**, **pandas-datareader**, and **wbgapi** packages in your terminal using `uv add beautifulsoup4`, and `uv add wbgapi` respectively. We'll also use two built-in packages, **textwrap** and **requests**.\n", "\n", "To kick off, let's import some of the packages we need (it's always good practice to import the packages you need at the top of a script or notebook)." ] @@ -82,11 +82,10 @@ "import textwrap\n", "\n", "import pandas as pd\n", + "import polars as pl\n", "import requests\n", "from bs4 import BeautifulSoup\n", - "from lets_plot import *\n", - "\n", - "LetsPlot.setup_html()" + "import lets_plot as lp" ] }, { @@ -95,7 +94,7 @@ "id": "f43a5237", "metadata": {}, "source": [ - "## Extracting Data from Files on the Internet using **pandas**\n", + "## Extracting Data from Files on the Internet using **polars**\n", "\n", "It's easy to read data from the internet once you have the url and file type. Here, for instance, is an example that reads in the 'storms' dataset, which is stored as a CSV file in a URL (we'll only grab the first 10 rows):" ] @@ -107,8 +106,8 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_csv(\n", - " \"https://vincentarelbundock.github.io/Rdatasets/csv/dplyr/storms.csv\", nrows=10\n", + "pl.read_csv(\n", + " \"https://vincentarelbundock.github.io/Rdatasets/csv/dplyr/storms.csv\", n_rows=10\n", ")" ] }, @@ -132,13 +131,13 @@ "\n", "An API has an 'endpoint', the base url, and then a URL that encodes the question. Let's see an example with the ONS API for which the endpoint is \"https://api.beta.ons.gov.uk/v1/\". The rest of the API has the form 'data?uri=' and then the long ID of both the timeseries (jp9z) and then the dataset (LMS), which is vacancies in the UK services sector.\n", "\n", - "The data that are returned by APIs are typically in JSON format, which looks a lot like a nested Python dictionary and its entries can be accessed in the same way--this is what is happening when getting the series' title in the example below. JSON is not good for analysis, so we'll use **pandas** to put the data into shape." + "The data that are returned by APIs are typically in JSON format, which looks a lot like a nested Python dictionary and its entries can be accessed in the same way--this is what is happening when getting the series' title in the example below. JSON is not good for analysis, so we'll use **polars** to put the data into shape." ] }, { "cell_type": "code", "execution_count": null, - "id": "c4226d67", + "id": "6107093c", "metadata": {}, "outputs": [], "source": [ @@ -147,57 +146,44 @@ "# Get the data from the ONS API:\n", "json_data = requests.get(url).json()\n", "\n", - "# Prep the data for a quick plot\n", "title = json_data[\"description\"][\"title\"]\n", + "\n", + "# Convert dates using string operations\n", "df = (\n", - " pd.DataFrame(pd.json_normalize(json_data[\"months\"]))\n", - " .assign(\n", - " date=lambda x: pd.to_datetime(x[\"date\"]),\n", - " value=lambda x: pd.to_numeric(x[\"value\"]),\n", - " )\n", - " .set_index(\"date\")\n", + " pl.DataFrame(json_data[\"months\"])\n", + " .with_columns([\n", + " # Add day to make it a valid date string\n", + " (pl.col(\"date\") + \"-01\").str.to_date(format=\"%Y %b-%d\").alias(\"date\"),\n", + " pl.col(\"value\").cast(pl.Float64).alias(\"value\"),\n", + " ])\n", + " .drop_nulls(\"date\")\n", + " .sort(\"date\")\n", ")\n", "\n", - "df[\"value\"].plot(title=title, ylim=(0, df[\"value\"].max() * 1.2), lw=3.0);" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "670ce0bb", - "metadata": {}, - "source": [ - "We've talked about *reading* APIs. You can also create your own to serve up data, models, whatever you like! This is an advanced topic and we won't cover it; but if you do need to, the simplest way is to use [Fast API](https://fastapi.tiangolo.com/). You can find some short video tutorials for Fast API [here](https://calmcode.io/fastapi/hello-world.html).\n", - "\n", - "### Pandas Datareader: an easier way to interact with (some) APIs\n", - "\n", - "Although it didn't take much code to get the ONS data, it would be even better if it was just a single line, wouldn't it? Fortunately there are some packages out there that make this easy, but it does depend on the API (and APIs come and go over time).\n", "\n", - "By far the most comprehensive library for accessing extra APIs is [**pandas-datareader**](https://pandas-datareader.readthedocs.io/en/latest/), which provides convenient access to:\n", "\n", - "- FRED\n", - "- Quandl\n", - "- World Bank\n", - "- OECD\n", - "- Eurostat\n", + "# Initialize the library\n", + "lp.LetsPlot.setup_html()\n", "\n", - "and more.\n", + "# Create plot using the alias\n", + "chart = lp.ggplot(df, lp.aes(x=\"date\", y=\"value\")) + \\\n", + " lp.geom_line(size=2.0, color=\"steelblue\") + \\\n", + " lp.ggtitle(title) + \\\n", + " lp.ylim(0, df[\"value\"].max() * 1.2) + \\\n", + " lp.theme_classic()\n", "\n", - "Let's see an example using FRED (the Federal Reserve Bank of St. Louis' economic data library). This time, let's look at the UK unemployment rate:" + "chart" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "bf758fb4", + "attachments": {}, + "cell_type": "markdown", + "id": "670ce0bb", "metadata": {}, - "outputs": [], "source": [ - "import pandas_datareader.data as web\n", - "\n", - "df_u = web.DataReader(\"LRHUTTTTGBM156S\", \"fred\")\n", + "We've talked about *reading* APIs. You can also create your own to serve up data, models, whatever you like! This is an advanced topic and we won't cover it; but if you do need to, the simplest way is to use [Fast API](https://fastapi.tiangolo.com/). You can find some short video tutorials for Fast API [here](https://calmcode.io/fastapi/hello-world.html).\n", "\n", - "df_u.plot(title=\"UK unemployment (percent)\", legend=False, ylim=(2, 6), lw=3.0);" + "### Accessing World Bank Data with wbgapi" ] }, { @@ -206,7 +192,9 @@ "id": "0613aefb", "metadata": {}, "source": [ - "And, because it's also a really useful one, let's see how to use the [**wbgapi**](https://pypi.org/project/wbgapi/) package to access World Bank data. (**pandas-datareader** used to provide a World Bank reader too, but it has not been actively maintained, so we prefer **wbgapi** for new work.)" + "While APIs can be accessed directly using tools like requests, some specialized libraries make working with structured datasets much easier. One such example is wbgapi, which provides a convenient interface for accessing World Bank data.\n", + "\n", + "Let’s look at an example using World Bank data on CO₂-equivalent emissions per capita:" ] }, { @@ -224,18 +212,23 @@ "import wbgapi as wb\n", "\n", "indicator_code = \"EN.GHG.ALL.PC.CE.AR5\"\n", + "\n", "df = (\n", - " wb.data.DataFrame(\n", - " indicator_code,\n", - " [\"USA\", \"CHN\", \"IND\", \"EAS\", \"ECS\"], # country and region codes\n", - " time=range(2019, 2020),\n", - " labels=True,\n", + " pl.from_pandas(\n", + " wb.data.DataFrame(\n", + " indicator_code,\n", + " [\"USA\", \"CHN\", \"IND\", \"EAS\", \"ECS\"],\n", + " time=range(2019, 2020),\n", + " labels=True,\n", + " ).reset_index()\n", " )\n", - " .rename(columns={\"Country\": \"country\", \"YR2019\": indicator_code})\n", - " .reset_index(drop=True)\n", + " .rename({\"Country\": \"country\"})\n", + " .with_columns(\n", + " pl.col(\"country\").map_elements(lambda x: textwrap.fill(x, 10))\n", + " )\n", + " .sort(indicator_code, descending=True)\n", ")\n", - "df[\"country\"] = df[\"country\"].apply(lambda x: textwrap.fill(x, 10)) # wrap long names\n", - "df = df.sort_values(indicator_code) # re-order\n", + "\n", "df.head()" ] }, @@ -246,19 +239,26 @@ "metadata": {}, "outputs": [], "source": [ - "(\n", - " ggplot(df, aes(x=\"country\", y=indicator_code))\n", - " + geom_bar(aes(fill=\"country\"), color=\"black\", alpha=0.8, stat=\"identity\")\n", - " + scale_fill_discrete()\n", - " + theme_minimal()\n", - " + theme(legend_position=\"none\")\n", - " + ggsize(600, 400)\n", - " + labs(\n", + "lp.LetsPlot.setup_html()\n", + "\n", + "country_order = df[\"country\"].to_list()\n", + "\n", + "plot = (\n", + " lp.ggplot(df, lp.aes(x=\"country\", y=indicator_code))\n", + " + lp.geom_bar(lp.aes(fill=\"country\"), color=\"black\", alpha=0.8, stat=\"identity\")\n", + " + lp.scale_x_discrete(limits=country_order)\n", + " + lp.scale_fill_discrete()\n", + " + lp.theme_minimal()\n", + " + lp.theme(legend_position=\"none\")\n", + " + lp.ggsize(600, 400)\n", + " + lp.labs(\n", " subtitle=\"Greenhouse gases (CO2-equivalent metric tons per capita, 2019)\",\n", " title=\"The USA leads the world on per-capita emissions\",\n", " y=\"\",\n", " )\n", - ")" + ")\n", + "\n", + "plot.show()" ] }, { diff --git a/workflow-help.quarto_ipynb_1 b/workflow-help.quarto_ipynb_1 new file mode 100644 index 0000000..f2d980b --- /dev/null +++ b/workflow-help.quarto_ipynb_1 @@ -0,0 +1,115 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Postscript: Getting Further Help {#sec-workflow-help}\n", + "\n", + "This book is not an island; there is no single resource that will allow you to master Python for Data Science. As you begin to apply the techniques described in this book to your own data, you will soon find questions that we do not answer. This section describes a few tips on how to get help, and to help you keep learning.\n", + "\n", + "## Resources\n", + "\n", + "Some other resources for learning are:\n", + "\n", + "- [The Python Data Science Handbook](https://jakevdp.github.io/PythonDataScienceHandbook/)\n", + "- [Real Python](https://realpython.com/), which has excellent short tutorials that cover Python more broadly (not just data science)\n", + "- [freeCodeCamp's Python courses](https://www.freecodecamp.org/news/search?query=data%20science%20python), though take care to select one that's at the right level for you\n", + "- [Coding for Economists](https://aeturrell.github.io/coding-for-economists), which has similar content to this book but is more in depth and aimed at analysts (particularly in economics)\n", + "\n", + "## Google is your friend\n", + "\n", + "If you get stuck, start with Google. Typically adding \"Python\" or \"Python Data Science\" (as the Python ecosystem goes *well* beyond data science) to a query is enough to restrict it to relevant results. Google is particularly useful for error messages. If you get an error message and you have no idea what it means, try googling it! Chances are that someone else has been confused by it in the past, and there will be help somewhere on the web.\n", + "\n", + "If Google doesn't help, try [Stack Overflow](http://stackoverflow.com). Start by spending a little time searching for an existing answer, including `[Python]` to restrict your search to questions and answers that use Python.\n", + "\n", + "## In the loop\n", + "\n", + "It's also helpful to keep an eye on the latest developments in data science. There are tons of data science newsletters out there, and we recommend keeping up with the Python data science community by following the (#pydata), (#datascience), and (#python) hashtags on Twitter.\n", + "\n", + "## Making a reprex (reproducible example)\n", + "\n", + "If your googling doesn't find anything useful, it's a really good idea prepare a minimal reproducible example or **reprex**.\n", + "\n", + "A good reprex makes it easier for other people to help you, and often you'll figure out the problem yourself in the course of making it. There are two parts to creating a reprex:\n", + "\n", + "- First, you need to make your code reproducible. This means that you need to capture everything, i.e., include any packages you used and create all necessary objects. The easiest way to make sure you've done this is to use the [**watermark**](https://github.com/rasbt/watermark) package alongside whatever else you are doing:" + ], + "id": "8f49fc2e" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import watermark.watermark as watermark\n", + "\n", + "print(watermark())\n", + "print(watermark(iversions=True, globals_=globals()))" + ], + "id": "eca8f7b4", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Second, you need to make it minimal. Strip away everything that is not directly related to your problem. This usually involves creating a much smaller and simpler Python object than the one you're facing in real life or even using built-in data.\n", + "\n", + "That sounds like a lot of work! And it can be, but it has a great payoff:\n", + "\n", + "- 80% of the time creating an excellent reprex reveals the source of your problem. It's amazing how often the process of writing up a self-contained and minimal example allows you to answer your own question.\n", + "\n", + "- The other 20% of time you will have captured the essence of your problem in a way that is easy for others to play with. This substantially improves your chances of getting help.\n", + "\n", + "There are several things you need to include to make your example reproducible: Python environment, required packages, data, and code.\n", + "\n", + "- **Python environment**--really just the Python version. This is covered by the first call to the **watermark** package.\n", + "\n", + "- **Packages** and their versions. These should be loaded at the top of the script, so it's easy to see which ones the example needs. By using **watermark** with the above configuration, you will also print the package versions. This is a good time to check that you're using the latest version of each package; it's possible you've discovered a bug that's been fixed since you installed or last updated the package.\n", + "\n", + "- **Data**: as others won't be able to easily download the data you're working with, it's often best to create a small amount of data from code that still have the same problem as you're finding with your actual data. Between **numpy** and **pandas**, it's quite easy to generate data from code; here's an example:" + ], + "id": "6ffdbbbe" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "df = pd.DataFrame(\n", + " data=np.reshape(range(36), (6, 6)),\n", + " index=[\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"],\n", + " columns=[\"col\" + str(i) for i in range(6)],\n", + " dtype=float,\n", + ")\n", + "df[\"random_normal\"] = np.random.normal(size=6)\n", + "df" + ], + "id": "6b14eb61", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- **Code**: copy and paste the minimal reproducible example code (including the packages, as noted above). Make sure you've used spaces and your variable names are concise, yet informative. Use comments to indicate where your problem lies. Do your best to remove everything that is not related to the problem. Finally, the shorter your code is, the easier it is to understand, and the easier it is to fix.\n", + "\n", + "Finish by checking that you have actually made a reproducible example by starting a fresh Python session and copying and pasting your reprex in." + ], + "id": "154142ef" + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "language": "python", + "display_name": "Python 3 (ipykernel)", + "path": "/Users/omagic/Documents/GitHub/python4DSpolars/.venv/share/jupyter/kernels/python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/workflow-packages-and-environments.quarto_ipynb_1 b/workflow-packages-and-environments.quarto_ipynb_1 new file mode 100644 index 0000000..fda4725 --- /dev/null +++ b/workflow-packages-and-environments.quarto_ipynb_1 @@ -0,0 +1,149 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Workflow: Packages and Environments {#sec-workflow-packages-and-environments}\n", + "\n", + "In this chapter, you're going to learn about packages and how to install them plus virtual coding environments that keep your packages isolated and your projects reproducible.\n", + "\n", + "## Packages\n", + "\n", + "### Introduction\n", + "\n", + "Packages (also called libraries) are key to extending the functionality of Python. It won't be long before you'll need to install some. There are packages for geoscience, for building websites, for analysing genetic data, for economics—pretty much for anything you can think of. Packages are typically not written by the core maintainers of the Python language but by enthusiasts, firms, researchers, academics, all sorts! Because anyone can write packages, they vary widely in their quality and usefulness. There are some that you'll be seeing them again and again.\n", + "\n", + "

Name a more iconic trio, I'll wait. pic.twitter.com/pGaLuUxQ3r

— Vicki Boykis (\\@vboykis) August 23, 2018
\n", + "\n", + "The three Python packages **numpy**, **pandas**, and **maplotlib**, which respectively provide numerical, data analysis, and plotting functionality, are ubiquitous. So many scripts begin by importing all three of them, as in the tweet above!\n", + "\n", + "There are typically two steps to using a new Python package:\n", + "\n", + "1. *install* the package on the command line (aka the terminal), eg using `uv add pandas`\n", + "\n", + "2. *import* the package into your Python session, eg using `import pandas as pd`\n", + "\n", + "When you issue an install command for a specific package, it is automatically downloaded from the internet and installed in the appropriate place on your computer. To install extra Python packages, you issue install commands to a text-based window called the \"terminal\".\n", + "\n", + "### The Command Line in Brief\n", + "\n", + "The *terminal* or *command line* or sometimes the *command prompt* was labelled 4 in the screenshot of Visual Studio Code from the chapter on @sec-introduction. The terminal is a text-based way to issue all kinds of commands to your computer (not just Python commands) and knowing a little bit about it is really useful for coding (and more) because managing packages, environments (which we haven't yet discussed), and version control (ditto) can all be done via the terminal. We'll come to these in due course in the chapter on @sec-command-line, but for now, a little background on what the terminal is and what it does.\n", + "\n", + "::: {.callout-note}\n", + "To open up the command line within Visual Studio Code, use the ⌃ + \\` keyboard shortcut (Mac) or ctrl + \\` (Windows/Linux), or click \"View > Terminal\".\n", + "\n", + "If you want to open up the command line independently of Visual Studio Code, search for \"Terminal\" on Mac and Linux, and \"Powershell\" on Windows.\n", + ":::\n", + "\n", + "Firstly, everything you can do by clicking on icons to launch programmes on your computer, you can also do via the terminal, also known as the command line. For many programmes, a lot of their functionality can be accessed using the command line, and other programmes *only* have a command line interface (CLI), including some that are used for data science.\n", + "\n", + "::: {.callout-tip}\n", + "The command line interacts with your operating system and is used to create, activate, or change Python installations.\n", + ":::\n", + "\n", + "Use Visual Studio Code to open a terminal window by clicking Terminal -> New Terminal on the list of commands at the very top of the window. If you have installed uv on your computer, your terminal should look something like this as your 'command prompt':\n", + "\n", + "```bash\n", + "your-username@your-computer current-directory %\n", + "```\n", + "\n", + "on Mac, and the same but with '%' replaced by '$' on linux, and (using Powershell)\n", + "\n", + "```powershell\n", + "PS C:\\Windows\\System32>\n", + "```\n", + "\n", + "on Windows.\n", + "\n", + "You can check that uv has successfully installed Python in your current project's folder by running\n", + "\n", + "```bash\n", + "uv run python --version\n", + "```\n", + "\n", + "For now, to at least try out the command line, let's use something that works across all three of the major operating systems. Type `uv run python` on the command prompt that came up in your new terminal window. You should see information about your installation of Python appear, including the version, followed by a Python prompt that looks like `>>>`. This is a kind of interactive Python session, in the terminal. It's much less rich than the one available in Visual Studio Code (it can't run scripts line-by-line, for example) but you can try `print('Hello World!')` and it will run, printing your message. To exit the terminal-based Python session, type `exit()` to go back to the regular command line.\n", + "\n", + "### Installing Packages\n", + "\n", + "To install extra Python packages, the default and easiest way is to use `uv add **packagename**`. There are over 330,000 Python packages on PyPI (the Python Package Index)! You can see what packages you have installed already by running `uv pip list` into the command line.\n", + "\n", + "`uv add ...` will install packages into the special Python environment in your current folder (it sits in a subdirectory called \".venv\" which will be hidden by default on most systems.) It's really helpful and good practice to have one Python environment per project, and **uv** does this automatically for you.\n", + "\n", + "::: {.callout-tip title=\"Exercise\"}\n", + "Try installing the **matplotlib**, **pandas**, **statsmodels**, and **skimpy** packages using `uv add`.\n", + ":::\n", + "\n", + "### Using Packages\n", + "\n", + "Once you have installed a package, you need to be able to use it! This is usually done via an import statement at the top of your script or Jupyter Notebook. For example, to bring in **pandas**, it's\n", + "\n", + "```python\n", + "import pandas as pd\n", + "```\n", + "\n", + "Why does Python do this? The idea of not just loading every package is to provide clarity over what function is being called from what package. It's also not necessary to load every package for every piece of analysis, and you often actually want to know what the *minimum* set of packages is to reproduce an analysis. Making the package imports explicit helps with all of that.\n", + "\n", + "You may also wonder why one doesn't just use `import pandas as pandas`. There's actually nothing stopping you doing this except i) it's convenient to have a shorter name and ii) there does tend to be a convention around imports, ie `pd` for **pandas** and `np` for **numpy**, and your code will be clearer to yourself and others if you follow the conventions.\n", + "\n", + "## Virtual Code Environments\n", + "\n", + "Virtual code environments allow you to isolate all of the packages that you're using to do analysis for one project from the set of packages you might need for a different project. They're an important part of creating a reproducible analytical pipeline but a key benefit is that others can reproduce the environment you used and it's best practice to have an isolated environment per project.\n", + "\n", + "To be more concrete, let's say you're using Python 3.9, **statsmodels**, and **pandas** for one project, project A. And, for project B, you need to use Python 3.10 with **numpy** and **scikit-learn**. Even with the same version of Python, best practice would be to have two separate virtual Python environments: environment A, with everything needed for project A, and environment B, with everything needed for project B. For the case where you're using different versions of Python, this isn't just best practice, it's essential.\n", + "\n", + "Many programming languages now come with an option to install packages and a version of the language in isolated environments. In Python, there are multiple tools for managing different environments. And, of those, the easiest to work with is probably [**uv**](https://docs.astral.sh/uv/).\n", + "\n", + "You can see all of the packages in the environment created in your current folder by running `uv pip list` on the command line. Here's an example of looking at the installed packages within this very book, filtering them just to the ones beginning with \"s\".\n", + "\n", + "```{bash}\n", + "uv run pip list | grep ^s\n", + "```\n", + "\n", + "### The pyproject.toml file in Python Environments\n", + "\n", + "You may have noticed that a file called `pyproject.toml` has been created." + ], + "id": "6b7f7364" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import toml\n", + "from rich import print_json\n", + "\n", + "print_json(data=toml.load(\"pyproject.toml\"))" + ], + "id": "acc9e141", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This lists all of the dependencies, and the version, of a **uv** Python project. There are lots of benefits to tracking what versions of packages you're using like this. One of the most important is that you can *share* projects with other people, and they can install them from these files too.\n", + "\n", + "As you install or remove packages, the `pyproject.toml` file changes in lockstep.\n", + "\n", + "Noe that Visual Studio Code shows which Python environment you are using when you open a Python script or Jupyter Notebook.\n", + "\n", + "![A typical user view in Visual Studio Code](https://github.com/aeturrell/coding-for-economists/blob/main/img/vscode_layout.png?raw=true)\n", + "\n", + "In the screenshot above, you can see the project-environment in two places: on the blue bar at the bottom of the screen, and (in 5), at the top right hand side of the interactive window. A similar top right indicator is present when you have a Jupyter Notebook open too." + ], + "id": "6ac222b7" + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "language": "python", + "display_name": "Python 3 (ipykernel)", + "path": "/Users/omagic/Documents/GitHub/python4DSpolars/.venv/share/jupyter/kernels/python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file From 89fdd7b5338746d5d3d94f08a4ee7b92d6f89339 Mon Sep 17 00:00:00 2001 From: Nwabueze Ugoh Date: Wed, 22 Apr 2026 19:49:47 +0100 Subject: [PATCH 12/34] Remove obsolete Quarto notebooks: visualise.quarto_ipynb_1, workflow-help.quarto_ipynb_1, and workflow-packages-and-environments.quarto_ipynb_1. --- visualise.quarto_ipynb_1 | 6 +- webscraping-and-apis.ipynb | 86 ++++++++++++------- workflow-help.quarto_ipynb_1 | 10 +-- ...w-packages-and-environments.quarto_ipynb_1 | 6 +- 4 files changed, 66 insertions(+), 42 deletions(-) diff --git a/visualise.quarto_ipynb_1 b/visualise.quarto_ipynb_1 index eb958d5..2ef6d2f 100644 --- a/visualise.quarto_ipynb_1 +++ b/visualise.quarto_ipynb_1 @@ -28,7 +28,7 @@ "\n", "The first of the three kinds is *exploratory data visualisation*, and it's the kind that you do when you're looking and data and trying to understand it. Just plotting the data is a really good strategy for getting a feel for any issues there might be. This is perhaps most famously demonstrated by Anscombe's quartet: four different datasets with the same mean, standard deviation, and correlation but very different data distributions." ], - "id": "2d4a4a09" + "id": "f3331573" }, { "cell_type": "code", @@ -96,7 +96,7 @@ "plt.suptitle(\"Anscombe's Quartet\")\n", "plt.show()" ], - "id": "c38c38fa", + "id": "64a0e7f6", "execution_count": null, "outputs": [] }, @@ -120,7 +120,7 @@ "\n", "You can find more information on the topic of communicating via data visualisations in the @sec-communicate-plots chapter." ], - "id": "88d12184" + "id": "30b9ff30" } ], "metadata": { diff --git a/webscraping-and-apis.ipynb b/webscraping-and-apis.ipynb index 6e8daf7..fc73aa9 100644 --- a/webscraping-and-apis.ipynb +++ b/webscraping-and-apis.ipynb @@ -267,15 +267,19 @@ "id": "b7bf16d7", "metadata": {}, "source": [ - "### The OECD API\n", + "### The Eurostat SDMX API\n", "\n", - "Sometimes it's convenient to use APIs directly, and, as an example, the OECD API comes with a LOT of complexity that direct access can take advantage of. The OECD API makes data available in both JSON and XML formats, and we'll use [**pandasdmx**](https://pandasdmx.readthedocs.io/) (aka the Statistical Data and Metadata eXchange (SDMX) package for the Python data ecosystem) to pull down the XML format data and turn it into a regular **pandas** data frame.\n", + "Sometimes it’s convenient to use APIs directly. The Eurostat API provides access to a massive repository of European statistical data using the SDMX (Statistical Data and Metadata eXchange) standard. While Eurostat offers multiple formats, using the SDMX-ML (XML) format via the sdmx1 library allows us to pull structured data into the Python ecosystem with high precision.\n", "\n", - "Now, key to using the OECD API is knowledge of its many codes: for countries, times, resources, and series. You can find some broad guidance on what codes the API uses [here](https://data.oecd.org/api/sdmx-ml-documentation/) but to find exactly what you need can be a bit tricky. Two tips are:\n", - "1. If you know what you're looking for is in a particular named dataset, eg \"QNA\" (Quarterly National Accounts), put `https://stats.oecd.org/restsdmx/sdmx.ashx/GetDataStructure/QNA/all?format=SDMX-ML` into your browser and look through the XML file; you can pick out the sub-codes and the countries that are available.\n", - "2. Browse around on https://stats.oecd.org/ and use Customise then check all the \"Use Codes\" boxes to see whatever your browsing's code names.\n", + "Key to using the Eurostat API is understanding the Data Structure Definition (DSD). Every dataset is essentially a multidimensional \"cube\" where each dimension (like Geography, Unit, or Frequency) has specific codes.\n", "\n", - "Let's see an example of this in action. We'd like to see the productivity (GDP per hour) data for a range of countries since 2010. We are going to be in the productivity resource (code \"PDB_LV\") and we want the USD current prices (code \"CPC\") measure of GDP per employed worker (code \"T_GDPEMP) from 2010 onwards (code \"startTime=2010\"). We'll grab this for some developed countries where productivity measurements might be slightly more comparable. The comments below explain what's happening in each step." + "To find the exact codes you need:\n", + "\n", + "The Data Browser: Browse the Eurostat Data Navigation Tree. Once you find a table (e.g., \"HICP - monthly data\"), the \"Dataset Code\" (like prc_hicp_manr) is shown in brackets.\n", + "\n", + "Positional Keys: Eurostat's REST API expects a \"key string\" where codes are placed in a specific order separated by dots (e.g., Freq.Unit.Item.Geo). If you know the order, you can \"slice\" the data cube directly.\n", + "\n", + "Let’s see an example of this in action. We want to see the Harmonised Index of Consumer Prices (HICP)—specifically the annual rate of change for all items—for Germany and France. We will use the resource prc_hicp_manr, requesting Monthly frequency (M), the Annual Rate of Change unit (RCH_A), and the \"All-items\" classification (CP00)." ] }, { @@ -285,17 +289,32 @@ "metadata": {}, "source": [ "```python\n", - "import pandasdmx as pdmx\n", - "# Tell pdmx we want OECD data\n", - "oecd = pdmx.Request(\"OECD\")\n", - "# Set out everything about the request in the format specified by the OECD API\n", - "data = oecd.data(\n", - " resource_id=\"PDB_LV\",\n", - " key=\"GBR+FRA+CAN+ITA+DEU+JPN+USA.T_GDPEMP.CPC/all?startTime=2010\",\n", - ").to_pandas()\n", - "\n", - "df = pd.DataFrame(data).reset_index()\n", - "df.head()\n", + "import sdmx\n", + "import polars as pl\n", + "\n", + "# Tell sdmx we want ESTAT data\n", + "client = sdmx.Client('ESTAT')\n", + "\n", + "# 2. Build the URL-style positional key\n", + "# Format: [Freq].[Unit].[Coicop].[Geo]\n", + "# We use '+' to join multiple countries (DE and FR)\n", + "resource_id = 'prc_hicp_manr'\n", + "key_string = 'M.RCH_A.CP00.DE+FR'\n", + "\n", + "# 3. Fetch the data directly\n", + "# 'startPeriod' limits the timeline to recent data\n", + "response = client.data(\n", + " resource_id=resource_id,\n", + " key=key_string,\n", + " params={'startPeriod': '2024-01'}\n", + ")\n", + "\n", + "# 4. Convert the SDMX-ML response to a Polars DataFrame\n", + "# We bridge through Pandas as sdmx1 is optimized for it\n", + "df_pd = sdmx.to_pandas(response).to_frame(name='value').reset_index()\n", + "df = pl.from_pandas(df_pd)\n", + "\n", + "print(df.head())\n", "```" ] }, @@ -305,13 +324,13 @@ "id": "e5cac233", "metadata": {}, "source": [ - "| | LOCATION | SUBJECT | MEASURE | TIME_PERIOD | value |\n", - "|--:|---------:|---------:|--------:|------------:|-------------:|\n", - "| 0 | CAN | T_GDPEMP | CPC | 2010 | 78848.604088 |\n", - "| 1 | CAN | T_GDPEMP | CPC | 2011 | 81422.364748 |\n", - "| 2 | CAN | T_GDPEMP | CPC | 2012 | 82663.028058 |\n", - "| 3 | CAN | T_GDPEMP | CPC | 2013 | 86368.582158 |\n", - "| 4 | CAN | T_GDPEMP | CPC | 2014 | 89617.632446 |" + "| | TIME_PERIOD | geo | unit | freq | coicop | value |\n", + "|--:|------------:|:----|:------|:-----|:-------|------:|\n", + "| 0 | 2024-01 | DE | RCH_A | M | CP00 | 3.1 |\n", + "| 1 | 2024-02 | DE | RCH_A | M | CP00 | 2.7 |\n", + "| 2 | 2024-03 | DE | RCH_A | M | CP00 | 2.3 |\n", + "| 3 | 2024-04 | DE | RCH_A | M | CP00 | 2.4 |\n", + "| 4 | 2024-05 | DE | RCH_A | M | CP00 | 2.8 |" ] }, { @@ -489,11 +508,11 @@ "source": [ "### Webscraping Tables\n", "\n", - "Often there are times when you don't actually want to scrape an entire webpage and all you want is the data from a *table* within the page. Fortunately, there is an easy way to scrape individual tables using the **pandas** package.\n", + "There are times when you don't need to scrape an entire webpage; you simply want the structured data from a specific table. While Polars is a high-performance data engine, it focuses on strict data formats (like Parquet or CSV) and does not natively include an HTML parser. However, we can easily bridge this gap by using Pandas to fetch the table and then converting it into a Polars DataFrame.\n", "\n", - "We will read data from a table on 'https://webscraper.io/test-sites/tables' using **pandas**. The function we'll use is `read_html()`, which returns a list of data frames of all the tables it finds when you pass it a URL. If you want to filter the list of tables, use the `match=` keyword argument with text that only appears in the table(s) you're interested in.\n", + "We will read data from 'https://webscraper.io/test-sites/tables' using `pd.read_html()`. This function scans the webpage and returns a list of all tables it finds as DataFrames. To target a specific table, we use the match= keyword argument with text that uniquely appears in the table we want—in this case, \"First Name\".\n", "\n", - "The example below shows how this works; looking at the website, we can see that the table we're interested in, has a 'First Name' column. Therefore we run:" + "Once captured, we convert the result to Polars using pl.from_pandas() to take advantage of Polars' superior query performance and expression API." ] }, { @@ -503,10 +522,15 @@ "metadata": {}, "outputs": [], "source": [ - "df_list = pd.read_html(\"https://webscraper.io/test-sites/tables\", match=\"First Name\")\n", + "import polars as pl\n", + "import pandas as pd\n", + "\n", + "\n", + "pd_list = pd.read_html(\"https://webscraper.io/test-sites/tables\", match=\"First Name\")\n", "# Retrieve first entry from list of data frames\n", - "df = df_list[0]\n", - "df.head()" + "df = pl.from_pandas(pd_list[0])\n", + "\n", + "print(df.head())" ] }, { @@ -515,7 +539,7 @@ "id": "31e49317", "metadata": {}, "source": [ - "This gives us the table neatly loaded into a **pandas** data frame ready for further use.\n", + "This gives us the table neatly loaded into a **polars** data frame ready for further use.\n", "\n", "If you get a '403' error, it means that the website has blocked **pandas** because it can see that you are engaged in web scraping. This is because some people web scrape irresponsibly, or because websites have provided other, preferred ways for you to obtain the data, eg via a download of the whole thing (think Wikipedia) or through an API. (If you really need to, [you can often get around the 403 error](https://stackoverflow.com/questions/43590153/http-error-403-forbidden-when-reading-html) though.)" ] diff --git a/workflow-help.quarto_ipynb_1 b/workflow-help.quarto_ipynb_1 index f2d980b..e7bebf8 100644 --- a/workflow-help.quarto_ipynb_1 +++ b/workflow-help.quarto_ipynb_1 @@ -35,7 +35,7 @@ "\n", "- First, you need to make your code reproducible. This means that you need to capture everything, i.e., include any packages you used and create all necessary objects. The easiest way to make sure you've done this is to use the [**watermark**](https://github.com/rasbt/watermark) package alongside whatever else you are doing:" ], - "id": "8f49fc2e" + "id": "22b3f9e0" }, { "cell_type": "code", @@ -48,7 +48,7 @@ "print(watermark())\n", "print(watermark(iversions=True, globals_=globals()))" ], - "id": "eca8f7b4", + "id": "a119501b", "execution_count": null, "outputs": [] }, @@ -72,7 +72,7 @@ "\n", "- **Data**: as others won't be able to easily download the data you're working with, it's often best to create a small amount of data from code that still have the same problem as you're finding with your actual data. Between **numpy** and **pandas**, it's quite easy to generate data from code; here's an example:" ], - "id": "6ffdbbbe" + "id": "c4ac60b4" }, { "cell_type": "code", @@ -87,7 +87,7 @@ "df[\"random_normal\"] = np.random.normal(size=6)\n", "df" ], - "id": "6b14eb61", + "id": "d1e4562c", "execution_count": null, "outputs": [] }, @@ -99,7 +99,7 @@ "\n", "Finish by checking that you have actually made a reproducible example by starting a fresh Python session and copying and pasting your reprex in." ], - "id": "154142ef" + "id": "4b75e409" } ], "metadata": { diff --git a/workflow-packages-and-environments.quarto_ipynb_1 b/workflow-packages-and-environments.quarto_ipynb_1 index fda4725..a5600ce 100644 --- a/workflow-packages-and-environments.quarto_ipynb_1 +++ b/workflow-packages-and-environments.quarto_ipynb_1 @@ -104,7 +104,7 @@ "\n", "You may have noticed that a file called `pyproject.toml` has been created." ], - "id": "6b7f7364" + "id": "8b889898" }, { "cell_type": "code", @@ -115,7 +115,7 @@ "\n", "print_json(data=toml.load(\"pyproject.toml\"))" ], - "id": "acc9e141", + "id": "688f09f1", "execution_count": null, "outputs": [] }, @@ -133,7 +133,7 @@ "\n", "In the screenshot above, you can see the project-environment in two places: on the blue bar at the bottom of the screen, and (in 5), at the top right hand side of the interactive window. A similar top right indicator is present when you have a Jupyter Notebook open too." ], - "id": "6ac222b7" + "id": "148595b3" } ], "metadata": { From af97f7149a9ce95b336ef676fec450a3fbe3eaa9 Mon Sep 17 00:00:00 2001 From: Uchenna Ugoh Date: Thu, 23 Apr 2026 19:13:15 +0100 Subject: [PATCH 13/34] Refactor code and structure to polars for improved readability and maintainability --- data-transform.ipynb | 296 ++- dataframe_illustration.svg | 4481 ++++++++++++++++++++++++++++++++++++ 2 files changed, 4621 insertions(+), 156 deletions(-) create mode 100644 dataframe_illustration.svg diff --git a/data-transform.ipynb b/data-transform.ipynb index 0465933..cb42a05 100644 --- a/data-transform.ipynb +++ b/data-transform.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "04dcb195", "metadata": {}, "outputs": [], @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "eff283e8", "metadata": {}, "outputs": [], @@ -266,43 +266,10 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "3958ddb5", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 7)
col0col1col2col3col4col5col6
i64i64i64i64i64i64str
012345"apple"
67891011"orange"
121314151617"pineapple"
181920212223"mango"
242526272829"kiwi"
303132333435"lemon"
" - ], - "text/plain": [ - "shape: (6, 7)\n", - "┌──────┬──────┬──────┬──────┬──────┬──────┬───────────┐\n", - "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 ┆ col6 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", - "╞══════╪══════╪══════╪══════╪══════╪══════╪═══════════╡\n", - "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ 4 ┆ 5 ┆ apple │\n", - "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ 10 ┆ 11 ┆ orange │\n", - "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ 16 ┆ 17 ┆ pineapple │\n", - "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ 22 ┆ 23 ┆ mango │\n", - "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 ┆ kiwi │\n", - "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ 34 ┆ 35 ┆ lemon │\n", - "└──────┴──────┴──────┴──────┴──────┴──────┴───────────┘" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import numpy as np\n", "\n", @@ -346,7 +313,7 @@ "id": "18124edd", "metadata": {}, "source": [ - "You can also access particular rows based on a predicate using `.row()` with the `by_predicate` parameter.\n" + "We can also access particular rows based on a predicate using `.row()` with the `by_predicate` parameter.\n" ] }, { @@ -381,6 +348,24 @@ "df.row(by_predicate=pl.col(\"col6\") == \"mango\", named=True)" ] }, + { + "cell_type": "markdown", + "id": "980b7be6", + "metadata": {}, + "source": [ + "We can also access rows using the `.slice()` function. As the function name implies, we get a slice of the DataFrame. we can use this to get a single row or a number of rows. To use this, we give it an offset - a start index, negative indexing is supported to index from the bottom of the DataFrame - and a length of the slice. This returns a DataFrame\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f7e8e892", + "metadata": {}, + "outputs": [], + "source": [ + "df.slice(-2, 2)" + ] + }, { "cell_type": "markdown", "id": "77f67ac2", @@ -585,7 +570,7 @@ "source": [ "### Creating New Columns\n", "\n", - "Let's now move on to creating new columns, either using new information or from existing columns. Given a data frame, `df`, creating a new column with the same value repeated is as easy as using square brackets with a string (text enclosed by quotation marks) in.\n" + "Let's now move on to creating new columns, either using new information or from existing columns. Given a data frame, `df`, creating a new column with the same value repeated is done by using `.with_columns()`, with an expression assigned to a column name. Here we use `pl.lit()` which returns an expression representing a literal value, 5 in our case.\n" ] }, { @@ -595,7 +580,7 @@ "metadata": {}, "outputs": [], "source": [ - "df = df.with_columns(pl.lit(5).alias(\"new_column0\"))\n", + "df = df.with_columns(new_column0=pl.lit(5))\n", "df" ] }, @@ -604,7 +589,7 @@ "id": "55bc84a5", "metadata": {}, "source": [ - "If we do the same operation again, but with a different right-hand side, it will overwrite what was already in that column. Let's see this with an example where we put different values in each position by assigning a list to the new column.\n" + "If we do the same operation again, but this time assigning a **_Series_** to the same column, it will overwrite what was already in that column. A **_Series_** repesents a single column in a Polars DataFrame. Let's see this with an example where we put different values in each position by assigning a list to the new column.\n" ] }, { @@ -627,7 +612,7 @@ "What happens if you try to use assignment where the right-hand side values are longer or shorter than the length of the data frame?\n", ":::\n", "\n", - "By passing a list within the square brackets, we can actually create more than one new column:\n" + "We can actually use `.with_columns` to create more than one new column:\n" ] }, { @@ -646,7 +631,7 @@ "id": "10792ddd", "metadata": {}, "source": [ - "Very often, you will want to create a new column that is the result of an operation on existing columns. There are a couple of ways to do this. The 'stand-alone' method works in a similar way to what we've just seen except that we refer to the data frame on the right-hand side of the assignment statement too:\n" + "Very often, you will want to create a new column or modify a column that is the result of an operation on existing columns. There are a couple of ways to do this. The 'stand-alone' method works in a similar way to what we've just seen except that we refer to the data frame on the right-hand side of the assignment statement too:\n" ] }, { @@ -656,8 +641,7 @@ "metadata": {}, "outputs": [], "source": [ - "df = df.with_columns(new_column3=pl.col(\"col0\") - pl.col(\"new_column0\"))\n", - "df" + "df.with_columns(new_column2=pl.col(\"col0\") - pl.col(\"new_column0\"))\n" ] }, { @@ -665,6 +649,7 @@ "id": "03172fa9", "metadata": {}, "source": [ + "We can use `.alias()` with an expression to assign column names, when creating new columns.\n", "The other way to do this involves an 'assign()' statement and is used when you wish to chain multiple steps together (like we saw earlier). These use a special syntax called a 'lambda' statement, which (here at least) just provides a way of specifying to **pandas** that we wish to perform the operation on every row. Below is an example using the flights data. You should note though that the word 'row' below is a dummy; you could replace it with any variable name (for example, `x`) but `row` makes what is happening a little bit clearer.\n" ] }, @@ -678,7 +663,7 @@ "\n", "flights.with_columns(\n", " (pl.col(\"dep_delay\") - pl.col(\"arr_delay\")).alias(\"gain\"),\n", - " (pl.col(\"distance\") / pl.col(\"air_time\") * 60).alias(\"speed\")\n", + " (pl.col(\"distance\") / pl.col(\"air_time\") * 60).alias(\"speed\"),\n", ")" ] }, @@ -728,46 +713,63 @@ }, { "cell_type": "markdown", - "id": "2b2a7be0", + "id": "a6fdfc17", "metadata": {}, "source": [ - "If you want to access particular rows at the same time, use the `.loc` access function:\n" + "We can also use `.select()` on the data frame to select columns, passing a single string to select a single column or an iterable, like a _list_, _positional arguments_ or _keyword arguments_, to select multiple columns. **Using _keyword arguments_ renames the columns in the output**\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "eabfd313", + "id": "1bc0cd22", "metadata": {}, "outputs": [], "source": [ - "df.loc[[\"a\", \"b\"], [\"col0\", \"new_column0\", \"col2\"]]" + "# selecting a single column\n", + "df.select(\"col0\")\n", + "\n", + "# Using positional arguments to select multiple columns\n", + "df.select(\"col0\", \"new_column0\", \"col2\")\n", + "\n", + "# Using keyword arguments to rename columns in the output\n", + "df.select(col1=\"col0\", col2=\"new_column0\", col3=\"col2\")" ] }, { "cell_type": "markdown", - "id": "c1b7db13", + "id": "a806be16", "metadata": {}, "source": [ - "And, just as with rows, we can access columns by their position using `.iloc` (where `:` stands in for 'any row').\n" + "Expressions are also accepted\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "b6ae1605", + "id": "ed447fb7", "metadata": {}, "outputs": [], "source": [ - "df.iloc[:, [0, 1]]" + "df.select(pl.col(\"col0\"), pl.col(\"new_column0\") + 2, pl.col(\"col2\") * 2)" ] }, { "cell_type": "markdown", - "id": "509dc236", + "id": "2b2a7be0", "metadata": {}, "source": [ - "There are other ways to access multiple columns that make use of slicing but we’ll leave that topic for another time.\n" + "If we want to access particular rows at the same time, we can chain `.filter()` or `.slice()` to the `.select()` function:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eabfd313", + "metadata": {}, + "outputs": [], + "source": [ + "df.select(\"col0\", \"new_column0\", \"col2\").slice(0, 2)" ] }, { @@ -775,46 +777,55 @@ "id": "17b928c8", "metadata": {}, "source": [ - "Sometimes, you'll want to select columns based on the _type_ of data that they hold. For this, **pandas** provides a function `.select_dtypes()`. Let's use this to select all columns with integers in the flights data.\n" + "Sometimes, we'll want to select columns based on the _type_ of data that they hold. For this, we can call **polars** data types with an expression inside `.select()`. Let's use this to select all columns with integers in the flights data.\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "62f578d1", + "id": "aed67406", "metadata": {}, "outputs": [], "source": [ - "flights.select_dtypes(\"int\")" + "flights.select(pl.col(pl.Int64))\n" ] }, { "cell_type": "markdown", - "id": "9aec778c", + "id": "8cb930af", "metadata": {}, "source": [ - "There are other occasions when you'd like to select columns based on criteria such as patterns in the _name_ of the column. Because Python has very good support for text, this is very possible but doesn't tend to be so built-in to **pandas** functions. The trick is to generate a list of column names that you want from the pattern you're interested in.\n", - "\n", - "Let's see a couple of examples. First, let's get all columns in our `df` data frame that begin with `\"new_...\"`. We'll generate a list of true and false values reflecting if each of the columns begins with \"new\" and then we'll pass those true and false values to `.loc`, which will only give columns for which the result was `True`. To show what's going on, we'll break it into two steps:\n" + "**polars** also provides a `selectors` module that we can use to select columns based on both data types and criteria such as patterns in the name of the column.\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "5aaae8bd", + "id": "62f578d1", "metadata": {}, "outputs": [], "source": [ - "print(\"The list of columns:\")\n", - "print(df.columns)\n", - "print(\"\\n\")\n", + "import polars.selectors as S\n", "\n", - "print(\"The list of true and false values:\")\n", - "print(df.columns.str.startswith(\"new\"))\n", - "print(\"\\n\")\n", + "# Select all integer columns\n", + "flights.select(S.integer())\n", "\n", - "print(\"The selection from the data frame:\")\n", - "df.loc[:, df.columns.str.startswith(\"new\")]" + "# Exclude string columns\n", + "flights.select(S.exclude(S.string()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4e486db", + "metadata": {}, + "outputs": [], + "source": [ + "# Select columns that contain \"delay\" in their name\n", + "flights.select(S.contains(\"delay\"))\n", + "\n", + "# Select columns that start with \"arr\"\n", + "flights.select(S.starts_with(\"arr\"))" ] }, { @@ -822,7 +833,7 @@ "id": "b514cbf4", "metadata": {}, "source": [ - "As well as `startswith()`, there are other commands like `endswith()`, `contains()`, `isnumeric()`, and `islower()`.\n" + "Other `selectors` commands include `ends_with()`, `by_index()`, `first()`, `last()`, `duration()`, `numeric()`, `boolean()` and more\n" ] }, { @@ -832,7 +843,7 @@ "source": [ "### Renaming Columns\n", "\n", - "There are three easy ways to rename columns, depending on what the context is. The first is to use the dedicated `rename()` function with an object called a dictionary. Dictionaries in Python consist of curly brackets with comma separated pairs of values where the first values maps into the second value. An example of a dictionary would be `{'old_col1': 'new_col1', 'old_col2': 'new_col2'}`. Let's see this in practice (but note that we are not 'saving' the resulting data frame, just showing it—to save it, you'd need to add `df = ` to the left-hand side of the code below).\n" + "We use the dedicated `rename()` function with a mapping, such as a dictionary or a lambda function. Dictionaries in Python consist of curly brackets with comma separated pairs of values where the first values maps into the second value. An example of a dictionary would be `{'old_col1': 'new_col1', 'old_col2': 'new_col2'}`. Let's see this in practice (but note that we are not 'saving' the resulting data frame, just showing it—to save it, you'd need to add `df = ` to the left-hand side of the code below).\n" ] }, { @@ -842,7 +853,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.rename(columns={\"col3\": \"letters\", \"col4\": \"names\", \"col6\": \"fruit\"})" + "df.rename({\"col3\": \"letters\", \"col4\": \"names\", \"col6\": \"fruit\"})" ] }, { @@ -850,7 +861,7 @@ "id": "0a673852", "metadata": {}, "source": [ - "The second method is for when you want to rename all of the columns. For that you simply set `df.columns` equal to the new set of columns that you'd like to have. For example, we might want to capitalise the first letter of each column using `str.capitalize()` and assign that to `df.columns`.\n" + "Using a lambda function, maps each column name as its argument, which you can then perform an operation on.\n" ] }, { @@ -860,26 +871,16 @@ "metadata": {}, "outputs": [], "source": [ - "df.columns = df.columns.str.capitalize()\n", - "df" + "df.rename(lambda column_name: column_name.upper())" ] }, { "cell_type": "markdown", - "id": "7a8b9660", - "metadata": {}, - "source": [ - "Finally, we might be interested in just replacing specific parts of column names. In this case, we can use `.str.replace()`. As an example, let's add the word `\"Original\"` ahead of the original columns:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3dd7606b", + "id": "aa80d44b", "metadata": {}, - "outputs": [], "source": [ - "df.columns.str.replace(\"Col\", \"Original_column\")" + "::: {.callout-tip}\n", + "A _lambda function_ is a small, anonymous function in Python that performs a single operation. It's a shorthand way to create a function without using `def`.\n" ] }, { @@ -911,7 +912,6 @@ "source": [ "df = pl.DataFrame(\n", " data=np.reshape(range(36), (6, 6)),\n", - " index=[\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"],\n", " columns=[\"col\" + str(i) for i in range(6)],\n", " dtype=float,\n", ")\n", @@ -947,46 +947,6 @@ "df.reindex(sorted(df.columns), axis=1)" ] }, - { - "cell_type": "markdown", - "id": "28e49605", - "metadata": {}, - "source": [ - "## Review of How to Access Rows, Columns, and Values\n", - "\n", - "With all of these different ways to access values in data frames, it can get confusing. These are the different ways to get the first column of a data frame (when that first column is called `column` and the data frame is `df`):\n", - "\n", - "- `df.column`\n", - "- `df[\"column\"]`\n", - "- `df.loc[:, \"column\"]`\n", - "- `df.iloc[:, 0]`\n", - "\n", - "Note that `:` means 'give me everything'! The ways to access rows are similar (here assuming the first row is called `row`):\n", - "\n", - "- `df.loc[\"row\", :]`\n", - "- `df.iloc[0, :]`\n", - "\n", - "And to access the first value (ie the value in first row, first column):\n", - "\n", - "- `df.column[0]`\n", - "- `df[\"column\"][0]`\n", - "- `df.iloc[0, 0]`\n", - "- `df.loc[\"row\", \"column\"]`\n", - "\n", - "In the above examples, square brackets are instructions about _where_ to grab bits from the data frame. They are a bit like an address system for values within a data frame. Square brackets _also_ denote lists though. So if you want to select _multiple_ columns or rows, you might see syntax like this:\n", - "\n", - "`df.loc[[\"row0\", \"row1\"], [\"column0\", \"column2\"]]`\n", - "\n", - "which picks out two rows and two columns via the lists `[\"row0\", \"row1\"]` and `[\"column0\", \"column2\"]`. Because there are lists alongside the usual system of selecting values, there are two sets of square brackets.\n", - "\n", - "::: {.callout-tip title=\"Tip\"}\n", - "\n", - "If you only want to remember one syntax for accessing rows and columns by name, use the pattern `df.loc[[\"row0\", \"row1\", ...], [\"col0\", \"col1\", ...]]`. This also works with a single row or a single column (or both).\n", - "\n", - "If you only want to remember one syntax for accessing rows and columns by position, use the pattern `df.iloc[[0, 1, ...], [0, 1, ...]]`. This also works with a single row or a single column (or both).\n", - ":::\n" - ] - }, { "cell_type": "markdown", "id": "f0e4c898", @@ -1026,7 +986,7 @@ "source": [ "## Grouping, changing the index, and applying summary statistics\n", "\n", - "So far you've learned about working with rows and columns. **pandas** gets even more powerful when you add in the ability to work with groups. Creating groups will often also mean a change of index. And because groups tend to imply an aggregation or pooling of data, they often go hand-in-hand with the application of a summary statistic.\n", + "So far you've learned about working with rows and columns. **polars** gets even more powerful when you add in the ability to work with groups. And because groups tend to imply an aggregation or pooling of data, they go hand-in-glove with the application of a summary statistic.\n", "\n", "The diagram below gives a sense of how these operations can proceed together. Note that the 'split' operation is achieved through grouping, while apply produces summary statistics. At the end, you get a data frame with a new index (one entry per group) in what is shown as the 'combine' step.\n", "\n", @@ -1034,19 +994,55 @@ "\n", "### Grouping and Aggregating\n", "\n", - "Let's take a look at creating a group using the `.groupby()` function followed by selecting a column and applying a summary statistic via an aggregation. Note that _aggregation_, via `.agg()`, always produces a new index because we have collapsed information down to the group-level (and the new index is made of those levels).\n", - "\n", - "The key point to remember is: use `.agg()` with `.groupby()` when you want your groups to become the new index.\n" + "Let's take a look at creating a group using the `.group_by()` function, then followed by the `.agg()` function for aggregation, selecting a column and applying a summary statistic via an aggregation.\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 92, "id": "62e540c0", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (12, 2)
monthdep_delay
i64f64
115.435362
1216.576688
210.816843
812.61104
106.243988
……
620.846332
110.036665
721.727787
313.227076
512.986859
" + ], + "text/plain": [ + "shape: (12, 2)\n", + "┌───────┬───────────┐\n", + "│ month ┆ dep_delay │\n", + "│ --- ┆ --- │\n", + "│ i64 ┆ f64 │\n", + "╞═══════╪═══════════╡\n", + "│ 11 ┆ 5.435362 │\n", + "│ 12 ┆ 16.576688 │\n", + "│ 2 ┆ 10.816843 │\n", + "│ 8 ┆ 12.61104 │\n", + "│ 10 ┆ 6.243988 │\n", + "│ … ┆ … │\n", + "│ 6 ┆ 20.846332 │\n", + "│ 1 ┆ 10.036665 │\n", + "│ 7 ┆ 21.727787 │\n", + "│ 3 ┆ 13.227076 │\n", + "│ 5 ┆ 12.986859 │\n", + "└───────┴───────────┘" + ] + }, + "execution_count": 92, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "(flights.group_by(\"month\")[[\"dep_delay\"]].mean())" + "flights.group_by(\"month\").agg(pl.mean(\"dep_delay\"))" ] }, { @@ -1054,19 +1050,9 @@ "id": "b003ea0d", "metadata": {}, "source": [ - "This now represents the mean departure delay by month. Notice that our index has changed! We now have month where we original had an index that was just the row number. The index plays an important role in grouping operations because it keeps track of the groups you have in the rest of your data frame.\n", + "This now represents the mean departure delay by month. The mechanics happenning here is that the DataFrame is grouped by each unique item in the _\"month\"_ column and then a mean summary statistic is derived on the _\"dep_delay\"_ column from each group.\n", "\n", - "Often, you might want to do multiple summary operations in one go. The most comprehensive syntax for this is via `.agg()`. We can reproduce what we did above using `.agg()`:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aaaca266", - "metadata": {}, - "outputs": [], - "source": [ - "(flights.group_by(\"month\")[[\"dep_delay\"]].agg(\"mean\"))" + "Other summary statistics can be derived from aggregations. Some common options are in the table below:\n" ] }, { @@ -1074,8 +1060,6 @@ "id": "520299c7", "metadata": {}, "source": [ - "where you pass in whatever aggregation you want. Some common options are in the table below:\n", - "\n", "| Aggregation | Description |\n", "| -------------------- | ------------------------------- |\n", "| `count()` | Number of items |\n", @@ -1099,9 +1083,9 @@ "outputs": [], "source": [ "(\n", - " flights.groupby([\"month\"]).agg(\n", - " mean_delay=(\"dep_delay\", \"mean\"),\n", - " count_flights=(\"dep_delay\", \"count\"),\n", + " flights.group_by(\"month\").agg(\n", + " mean_delay=pl.mean(\"dep_delay\"),\n", + " count_flights=pl.count(\"dep_delay\"),\n", " )\n", ")" ] @@ -1121,7 +1105,7 @@ "source": [ "### Grouping by multiple variables\n", "\n", - "This is as simple as passing `.groupby()` a list representing multiple columns instead of a string representing a single column.\n" + "This is as simple as passing `.group_by()` a list representing multiple columns instead of a string representing a single column.\n" ] }, { diff --git a/dataframe_illustration.svg b/dataframe_illustration.svg new file mode 100644 index 0000000..ce9dd66 --- /dev/null +++ b/dataframe_illustration.svg @@ -0,0 +1,4481 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 3391c7dba80295342fc02194470dfe09c799aaed Mon Sep 17 00:00:00 2001 From: Uchenna Ugoh Date: Thu, 23 Apr 2026 20:47:28 +0100 Subject: [PATCH 14/34] Refactor code and structure to polars --- data-transform.ipynb | 113 +- dataframe_illustration.svg | 4714 ++---------------------------------- 2 files changed, 264 insertions(+), 4563 deletions(-) diff --git a/data-transform.ipynb b/data-transform.ipynb index cb42a05..7f707d0 100644 --- a/data-transform.ipynb +++ b/data-transform.ipynb @@ -183,7 +183,7 @@ "Note: If you're coming from **pandas**, be aware that **polars** does not use an index column and each row is indexed by its integer position in the table.\n", ":::\n", "\n", - "![](https://pandas.pydata.org/docs/_images/01_table_dataframe.svg)\n" + "![](./dataframe_illustration.svg)\n" ] }, { @@ -650,7 +650,7 @@ "metadata": {}, "source": [ "We can use `.alias()` with an expression to assign column names, when creating new columns.\n", - "The other way to do this involves an 'assign()' statement and is used when you wish to chain multiple steps together (like we saw earlier). These use a special syntax called a 'lambda' statement, which (here at least) just provides a way of specifying to **pandas** that we wish to perform the operation on every row. Below is an example using the flights data. You should note though that the word 'row' below is a dummy; you could replace it with any variable name (for example, `x`) but `row` makes what is happening a little bit clearer.\n" + "We can chain multiple expressions together with `.with_columns()`, which would create multiple new columns with the names assigned to `.alias()`.\n" ] }, { @@ -900,7 +900,7 @@ "\n", "The simplest way to re-order (all) columns is to create a new list of their names with them in the order that you'd like them: but be careful you don't forget any columns that you'd like to keep!\n", "\n", - "Let's see an example with a fresh version of the fake data from earlier. We'll put all of the odd-numbered columns first, in descending order, then the even similarly.\n" + "Let's see an example with a fresh version of the fake data from earlier. We'll put all of the odd-numbered columns first, in descending order, then the even similarly using `.select()`.\n" ] }, { @@ -912,8 +912,7 @@ "source": [ "df = pl.DataFrame(\n", " data=np.reshape(range(36), (6, 6)),\n", - " columns=[\"col\" + str(i) for i in range(6)],\n", - " dtype=float,\n", + " schema=[\"col\" + str(i) for i in range(6)]\n", ")\n", "df" ] @@ -921,11 +920,11 @@ { "cell_type": "code", "execution_count": null, - "id": "b9a409ac", + "id": "3c2029cc", "metadata": {}, "outputs": [], "source": [ - "df = df[[\"col5\", \"col3\", \"col1\", \"col4\", \"col2\", \"col0\"]]\n", + "df = df.select([\"col5\", \"col3\", \"col1\", \"col4\", \"col2\", \"col0\"])\n", "df" ] }, @@ -934,7 +933,7 @@ "id": "dd91d87a", "metadata": {}, "source": [ - "Of course, this is quite tedious if you have lots of columns! There are methods that can help make this easier depending on your context. Perhaps you'd just liked to sort the columns in order? This can be achieved by combining `sorted()` and the `reindex()` command (which works for rows or columns) with `axis=1`, which means the second axis (i.e. columns).\n" + "Of course, this is quite tedious if you have lots of columns! There are methods that can help make this easier depending on your context. Perhaps you'd just liked to sort the columns in order? This can be achieved by combining `sorted()` and the `.select()` function, passing the DataFrame `.columns`.\n" ] }, { @@ -944,7 +943,11 @@ "metadata": {}, "outputs": [], "source": [ - "df.reindex(sorted(df.columns), axis=1)" + "# Alphabetical order\n", + "df.select(sorted(df.columns))\n", + "\n", + "# Reverse alphabetical order\n", + "df.select(sorted(df.columns, reverse=True))" ] }, { @@ -954,29 +957,13 @@ "source": [ "### Column and Row Exercises\n", "\n", - "1. Compare `air_time` with `arr_time - dep_time`. What do you expect to see? What do you see What do you need to do to fix it?\n", + "1. Compare `air_time` with `arr_time - dep_time`. What do you expect to see? What do you see? What do you need to do to fix it?\n", "\n", "2. Compare `dep_time`, `sched_dep_time`, and `dep_delay`. How would you expect those three numbers to be related?\n", "\n", "3. Brainstorm as many ways as possible to select `dep_time`, `dep_delay`, `arr_time`, and `arr_delay` from `flights`.\n", "\n", - "4. What happens if you include the name of a row or column multiple times when trying to select them?\n", - "\n", - "5. What does the `.isin()` function do in the following?\n", - "\n", - " ```python\n", - " flights.columns.isin([\"year\", \"month\", \"day\", \"dep_delay\", \"arr_delay\"])\n", - " ```\n", - "\n", - "6. Does the result of running the following code surprise you?\n", - " How do functions like `str.contains` deal with case by default?\n", - " How can you change that default?\n", - "\n", - " ```python\n", - " flights.loc[:, flights.columns.str.contains(\"TIME\")]\n", - " ```\n", - "\n", - " (Hint: you can use help even on functions that apply to data frames, eg use `help(flights.columns.str.contains)`)\n" + "4. What happens if you include the name of a row or column multiple times when trying to select them?\n" ] }, { @@ -984,7 +971,7 @@ "id": "a3c837e4", "metadata": {}, "source": [ - "## Grouping, changing the index, and applying summary statistics\n", + "## Grouping and applying summary statistics\n", "\n", "So far you've learned about working with rows and columns. **polars** gets even more powerful when you add in the ability to work with groups. And because groups tend to imply an aggregation or pooling of data, they go hand-in-glove with the application of a summary statistic.\n", "\n", @@ -999,50 +986,12 @@ }, { "cell_type": "code", - "execution_count": 92, + "execution_count": null, "id": "62e540c0", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (12, 2)
monthdep_delay
i64f64
115.435362
1216.576688
210.816843
812.61104
106.243988
……
620.846332
110.036665
721.727787
313.227076
512.986859
" - ], - "text/plain": [ - "shape: (12, 2)\n", - "┌───────┬───────────┐\n", - "│ month ┆ dep_delay │\n", - "│ --- ┆ --- │\n", - "│ i64 ┆ f64 │\n", - "╞═══════╪═══════════╡\n", - "│ 11 ┆ 5.435362 │\n", - "│ 12 ┆ 16.576688 │\n", - "│ 2 ┆ 10.816843 │\n", - "│ 8 ┆ 12.61104 │\n", - "│ 10 ┆ 6.243988 │\n", - "│ … ┆ … │\n", - "│ 6 ┆ 20.846332 │\n", - "│ 1 ┆ 10.036665 │\n", - "│ 7 ┆ 21.727787 │\n", - "│ 3 ┆ 13.227076 │\n", - "│ 5 ┆ 12.986859 │\n", - "└───────┴───────────┘" - ] - }, - "execution_count": 92, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "flights.group_by(\"month\").agg(pl.mean(\"dep_delay\"))" + "outputs": [], + "source": [ + "flights.group_by(\"month\").agg(pl.col(\"dep_delay\").mean())" ] }, { @@ -1062,13 +1011,14 @@ "source": [ "| Aggregation | Description |\n", "| -------------------- | ------------------------------- |\n", - "| `count()` | Number of items |\n", + "| `count()` | Number of non-null items |\n", + "| `len()` | Number of all items |\n", "| `first()`, `last()` | First and last item |\n", "| `mean()`, `median()` | Mean and median |\n", "| `min()`, `max()` | Minimum and maximum |\n", "| `std()`, `var()` | Standard deviation and variance |\n", "| `mad()` | Mean absolute deviation |\n", - "| `prod()` | Product of all items |\n", + "| `product()` | Product of all items |\n", "| `sum()` | Sum of all items |\n", "| `value_counts()` | Counts of unique values |\n", "\n", @@ -1082,11 +1032,10 @@ "metadata": {}, "outputs": [], "source": [ - "(\n", - " flights.group_by(\"month\").agg(\n", - " mean_delay=pl.mean(\"dep_delay\"),\n", - " count_flights=pl.count(\"dep_delay\"),\n", - " )\n", + "# Multiple aggregations using polars' syntactic sugar (shorthand) for mean and count\n", + "flights.group_by(\"month\").agg(\n", + " mean_delay=pl.mean(\"dep_delay\"),\n", + " count_flights=pl.count(\"dep_delay\"),\n", ")" ] }, @@ -1105,7 +1054,7 @@ "source": [ "### Grouping by multiple variables\n", "\n", - "This is as simple as passing `.group_by()` a list representing multiple columns instead of a string representing a single column.\n" + "This is as simple as passing `.group_by()` a list or multiple strings representing columns instead of a single string representing a single column.\n" ] }, { @@ -1115,9 +1064,9 @@ "metadata": {}, "outputs": [], "source": [ - "month_year_delay = flights.group_by([\"month\", \"year\"]).agg(\n", - " mean_delay=(\"dep_delay\", \"mean\"),\n", - " count_flights=(\"dep_delay\", \"count\"),\n", + "month_year_delay = flights.group_by(\"month\", \"year\").agg(\n", + " mean_delay=pl.mean(\"dep_delay\"),\n", + " count_flights=pl.count(\"dep_delay\"),\n", ")\n", "month_year_delay" ] @@ -1173,7 +1122,7 @@ "source": [ "### Groupby Exercises\n", "\n", - "1. Which carrier has the worst delays? Challenge: can you disentangle the effects of bad airports vs. bad carriers? Why/why not? (Hint: think about `flights.groupby([\"carrier\", \"dest\"]).count()`)\n", + "1. Which carrier has the worst delays? Challenge: can you disentangle the effects of bad airports vs. bad carriers? Why/why not? (Hint: think about `flights.group_by([\"carrier\", \"dest\"])`)\n", "\n", "2. Find the most delayed flight to each destination.\n", "\n", diff --git a/dataframe_illustration.svg b/dataframe_illustration.svg index ce9dd66..2c35021 100644 --- a/dataframe_illustration.svg +++ b/dataframe_illustration.svg @@ -1,4481 +1,233 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 032cd4d0c0351db5e0168726a4a9b282a21ad36c Mon Sep 17 00:00:00 2001 From: Uchenna Ugoh Date: Thu, 23 Apr 2026 20:51:25 +0100 Subject: [PATCH 15/34] Minor edit to data-transform.ipynb --- data-transform.ipynb | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/data-transform.ipynb b/data-transform.ipynb index 7f707d0..5e48955 100644 --- a/data-transform.ipynb +++ b/data-transform.ipynb @@ -1071,50 +1071,6 @@ "month_year_delay" ] }, - { - "cell_type": "markdown", - "id": "22e89b7e", - "metadata": {}, - "source": [ - "You might have noticed that this time we have a multi-index (that is, an index with more than one column). That's because we asked for something with multiple groups, and the index tracks what's going on within each group: so we need more than one dimension of index to do this efficiently.\n" - ] - }, - { - "cell_type": "markdown", - "id": "b3e1ea60", - "metadata": {}, - "source": [ - "### Grouping and Transforming\n", - "\n", - "You may not always want to change the index to reflect new groups when performing computations at the group level.\n", - "\n", - "The key point to remember is: use `.transform()` with `.groupby()` when you want to perform computations on your groups but you want to go back to the original index.\n", - "\n", - "Let's say we wanted to express the arrival delay, `\"arr_del\"`, of each flight as a fraction of the worst arrival delay in each month.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f96fcf41", - "metadata": {}, - "outputs": [], - "source": [ - "flights[\"max_delay_month\"] = flights.groupby(\"month\")[\"arr_delay\"].transform(\"max\")\n", - "flights[\"delay_frac_of_max\"] = flights[\"arr_delay\"] / flights[\"max_delay_month\"]\n", - "flights[\n", - " [\"year\", \"month\", \"day\", \"arr_delay\", \"max_delay_month\", \"delay_frac_of_max\"]\n", - "].head()" - ] - }, - { - "cell_type": "markdown", - "id": "3880ece9", - "metadata": {}, - "source": [ - "Note that the first few entries of `\"max_delay_month\"` are all the same because the month is the same for those entries, but the delay fraction changes with each row.\n" - ] - }, { "cell_type": "markdown", "id": "1954cb3b", From c01599511aa3ae4c54ee1dde4b296731271aedec Mon Sep 17 00:00:00 2001 From: Uchenna Ugoh Date: Thu, 23 Apr 2026 21:12:10 +0100 Subject: [PATCH 16/34] Remove unnecessary output and reset execution count in data visualisation notebook --- data-visualise.ipynb | 45 ++------------------------------------------ 1 file changed, 2 insertions(+), 43 deletions(-) diff --git a/data-visualise.ipynb b/data-visualise.ipynb index 81e2e28..2ee5cb2 100644 --- a/data-visualise.ipynb +++ b/data-visualise.ipynb @@ -62,51 +62,10 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "a86fb211", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "from lets_plot import *\n", "from palmerpenguins import load_penguins\n", From abdca1ff4bdfd1f336a88b85778ea0766ba7befc Mon Sep 17 00:00:00 2001 From: Uchenna Ugoh Date: Thu, 23 Apr 2026 21:14:05 +0100 Subject: [PATCH 17/34] Reformat YAML workflow for improved readability and consistency --- .github/workflows/tests.yml | 104 ++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cac3d2d..cbb2d2e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,59 +1,59 @@ name: tests on: - push: - branches: [main] - pull_request: + push: + branches: [main] + pull_request: concurrency: - group: tests-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: tests-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - uses: pre-commit/action@v3.0.1 - - build: - runs-on: ubuntu-latest - steps: - - name: Check out the repository - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - - name: Install uv - uses: astral-sh/setup-uv@v4 - - - name: Set up Python - run: uv python install - - - name: Install dependencies - run: uv sync --all-extras --dev - - - name: Install Quarto - uses: quarto-dev/quarto-actions/setup@v2 - with: - version: "1.5.57" - - - name: set timezone - run: | - TZ="Europe/London" && - sudo ln -snf /usr/share/zoneinfo/$TZ /etc/localtime - - - name: install linux deps - run: | - sudo apt-get -y install openssl graphviz nano texlive graphviz-dev unzip build-essential - - - name: build the book - run: | - uv run quarto render --execute - - - name: success - run: | - echo "Success in building book without errors!" + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - uses: pre-commit/action@v3.0.1 + + build: + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Set up Python + run: uv python install + + - name: Install dependencies + run: uv sync --all-extras --dev + + - name: Install Quarto + uses: quarto-dev/quarto-actions/setup@v2 + with: + version: "1.5.57" + + - name: set timezone + run: | + TZ="Europe/London" && + sudo ln -snf /usr/share/zoneinfo/$TZ /etc/localtime + + - name: install linux deps + run: | + sudo apt-get -y install openssl graphviz nano texlive graphviz-dev unzip build-essential + + - name: build the book + run: | + uv run quarto render --no-execute + + - name: success + run: | + echo "Success in building book without errors!" From 18fb95acfcab3c3676ce2ffc7502f80e5002d90f Mon Sep 17 00:00:00 2001 From: Uchenna Ugoh Date: Thu, 23 Apr 2026 21:29:45 +0100 Subject: [PATCH 18/34] Update workflow and notebooks for improved code execution and style consistency - Changed `quarto render` command in GitHub Actions from `--no-execute` to `--execute` for proper rendering. - Enhanced readability in `data-transform.ipynb` by removing unnecessary line breaks and simplifying code expressions. - Added missing import statement for `polars` in `data-visualise.ipynb`. - Improved code formatting and consistency across various cells in both notebooks. --- .github/workflows/tests.yml | 2 +- data-transform.ipynb | 25 ++++++++++++------------- data-visualise.ipynb | 2 +- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cbb2d2e..ad7aae3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -52,7 +52,7 @@ jobs: - name: build the book run: | - uv run quarto render --no-execute + uv run quarto render --execute - name: success run: | diff --git a/data-transform.ipynb b/data-transform.ipynb index 5e48955..c20ac1c 100644 --- a/data-transform.ipynb +++ b/data-transform.ipynb @@ -72,6 +72,7 @@ "outputs": [], "source": [ "import io\n", + "\n", "import requests\n", "\n", "url = \"https://raw.githubusercontent.com/byuidatascience/data4python4ds/master/data-raw/flights/flights.csv\"\n", @@ -163,9 +164,7 @@ "metadata": {}, "outputs": [], "source": [ - "flights.with_columns(\n", - " pl.col(\"time_hour\").str.to_datetime()\n", - ")" + "flights.with_columns(pl.col(\"time_hour\").str.to_datetime())" ] }, { @@ -235,7 +234,9 @@ "metadata": {}, "outputs": [], "source": [ - "flights.filter(pl.col(\"dest\") == \"IAH\").group_by([\"year\", \"month\", \"day\"]).agg(pl.col(\"arr_delay\").mean())" + "flights.filter(pl.col(\"dest\") == \"IAH\").group_by([\"year\", \"month\", \"day\"]).agg(\n", + " pl.col(\"arr_delay\").mean()\n", + ")" ] }, { @@ -277,7 +278,9 @@ " data=np.reshape(range(36), (6, 6)),\n", " schema=[\"col\" + str(i) for i in range(6)],\n", ")\n", - "df.insert_column(6, pl.Series(\"col6\", [\"apple\", \"orange\", \"pineapple\", \"mango\", \"kiwi\", \"lemon\"]))\n", + "df.insert_column(\n", + " 6, pl.Series(\"col6\", [\"apple\", \"orange\", \"pineapple\", \"mango\", \"kiwi\", \"lemon\"])\n", + ")\n", "df" ] }, @@ -383,9 +386,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.filter(\n", - " (pl.col(\"col6\") == \"kiwi\") | (pl.col(\"col6\") == \"pineapple\")\n", - ")" + "df.filter((pl.col(\"col6\") == \"kiwi\") | (pl.col(\"col6\") == \"pineapple\"))" ] }, { @@ -641,7 +642,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.with_columns(new_column2=pl.col(\"col0\") - pl.col(\"new_column0\"))\n" + "df.with_columns(new_column2=pl.col(\"col0\") - pl.col(\"new_column0\"))" ] }, { @@ -660,7 +661,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "flights.with_columns(\n", " (pl.col(\"dep_delay\") - pl.col(\"arr_delay\")).alias(\"gain\"),\n", " (pl.col(\"distance\") / pl.col(\"air_time\") * 60).alias(\"speed\"),\n", @@ -787,7 +787,7 @@ "metadata": {}, "outputs": [], "source": [ - "flights.select(pl.col(pl.Int64))\n" + "flights.select(pl.col(pl.Int64))" ] }, { @@ -911,8 +911,7 @@ "outputs": [], "source": [ "df = pl.DataFrame(\n", - " data=np.reshape(range(36), (6, 6)),\n", - " schema=[\"col\" + str(i) for i in range(6)]\n", + " data=np.reshape(range(36), (6, 6)), schema=[\"col\" + str(i) for i in range(6)]\n", ")\n", "df" ] diff --git a/data-visualise.ipynb b/data-visualise.ipynb index 2ee5cb2..488a1db 100644 --- a/data-visualise.ipynb +++ b/data-visualise.ipynb @@ -67,9 +67,9 @@ "metadata": {}, "outputs": [], "source": [ + "import polars as pl\n", "from lets_plot import *\n", "from palmerpenguins import load_penguins\n", - "import polars as pl\n", "\n", "LetsPlot.setup_html()" ] From d174e8036de59044bdc995567749c01a199b012c Mon Sep 17 00:00:00 2001 From: Uchenna Ugoh Date: Thu, 23 Apr 2026 23:20:20 +0100 Subject: [PATCH 19/34] Refactor webscraping-and-apis notebook: - Import `lets_plot` at the beginning for clarity. - Adjust DataFrame column operations for better readability. - Simplify plot creation by using parentheses for chaining. - Remove unnecessary imports and whitespace for cleaner code. --- data/bake_sale.xlsx | Bin 4969 -> 6218 bytes databases.ipynb | 4 +- rectangling.ipynb | 3 +- spreadsheets.ipynb | 1276 +++++++++++++++++++++++------------- webscraping-and-apis.ipynb | 35 +- 5 files changed, 836 insertions(+), 482 deletions(-) diff --git a/data/bake_sale.xlsx b/data/bake_sale.xlsx index cf475a2cfc9b5802745eb2c8fbd76dae562d3308..e122900cc12c6d5ee3e5bbbecc52b1a202f53865 100644 GIT binary patch literal 6218 zcmZ{J1z6MX*Z$}pE#2J=Mk6soa)dM@AdZp_0TGY}rBjepBm_iyAR!@0gT#OdO30)f zor>@H`g>o0^8fy~>$B(DuAO_=bD#6MpL3rX>f+$i0000&%%u##eNRbcg9QLk-~j;C zm`|+LJU#qi9)7kaL0&Ll8__^_x9W^Bk8W|&K6LsK(De09rdj)3UMt|a@28YLR(5XD zM6GmwAn9VL2Z{4DdRT;R^~}-nd7c$RBBD{-jKWaiS-riq9g)R)M$>*?h@S1o;4C{D zGolsYRX3aueeVyaom+Pw5&2sWThhJ=`@#&vYkW5x^d>{Vrd*>0A&$EJA^Is!CkyF! zxSw=HqKg~Y0`wrlSXf`L0CCXbOZARjm-TITOs3>g`-HLj;C`lQC;xg@ijk9bWad$z z8eRe$H#nyg3)vwfpIb!BO-;F6Szhs2h>x!;Xw{^U#azyAGV^7Y3gb|&m4q{?fMErn zHlb0^;=WOnAJzr==X|5d5KeD**}g<9#bA>Itf+2c_8Hzy_23VwM?|ft9k)=SNt_>k z_8?ym?{)iUPp2VB>JxX+o^+n_p#fL$&yGks$uk8TctfmNgXdR4N|pA0mE-aEGU&HQ zqXeGl`ehQszp`QY5ZbkY$$}m>06>CSZGB*FzG9-6uhnTIIyfMTN9YJN#oFRXQ$!pX zsZ05Qx`JGsw_bURmpaX;+0&DPMyT=N3n)FuvxC90J?Cf1BLCO$$Z(ixp~7`R@I-_GDIg-aDl`P&hjTu zx)sgvh|d~^Hhi)>QBwuDEX1xu@z+3^8{BGho6SiM=N8?kk}G-RBVnSkj>TU-9ck*l$(>Jbmun@$|fRS)%`8sLQoSoI)KPeGxeu zETSOtGUG!_6y&x085YFKnzxC&0a7&RVga7S}%ICA0ZM)0@ytknKpLgkOC za34^+p{(%=kJWtN-yP-kV? zufETW^|o#4gS@>D$^T|ZA@cZFD&`grzySd0e=~H+jjuBd=I1N+&jPw6DKTBwph}$P z(Xq^~GI&7&FJ_k!n?4cLDh}#6Vv$whI!I}f^hI|b)6fi%k_4b8_Vb-4E5mGtj_j%+ z`MX}UUihu=jMwE?JZl0_*?E?H{sPuhDZ9VyPEA9P_^ub7z9D<$pgu0WlIR=4870E_ zt;EKjJXeu67bD~Zw6`;@!1kz}^)Ff=1dZ$5I7Dh5 zW}#dvKA@QH;J-?+jUCV*)+hkxe5PgI(sP7**cL^1t7_wX8J20%_^~reAc3c3lGnRP zrg5FY48jvo9o*M*y}Z>McTR84<7Ik~>(0AjKy&*+J#l*6hH%?%ZWYbS<|0Z}@F=6| zWN%?OWNDSU_gT>d=ahA9)XhM@C{|l_E_n;b8oIen>Z7x`_JUfOp>KhdEQLiYTz-mb zwyeeP-B#PHmW*<(T!yU5s48$m)7RQ>63UC*-y|8e?A5Bp}LDi6=sY8cbWW zt`(GcXVJnZLy-?1VZ26ul554JvGu1x(7^-Pp26Q~G#>tj*2Sce4^!ymf71BBC7zh7 zZ_p!78iUsQ5&I!vhTNU@*$i}9R}b>2cXqVc8{zC?%p<_^i?!?~!6(_d8VBXQO=0tX z&02#7C}X~wM0|L$XkyBP!k+WBr47cX`?pA5C-gCQjCZ|)w^FMV}4KA?1qaq0rM zwj5q|Xd_}x@V+jMxS8)pLi4tyr!)Emb}4-S81jHdPER|R^%spBV$shS5A?)|g4QX$ zCX1*luS8PmF7jbCo5-hF_2I4f90ArF#R0?#x!S;oY@i&AQzACuT^5kKu5ur2+A4`;5jaiLX-IdW z#PjbKeehI+{Sb{!KHghnr6D;ZcHhZRGawyWK6B-O*OnO1`vMcS6cD-1WOA$8&I4uH z^6If^uf5A!5~FBk)R(IYU~~ERofY|SzO*!CwQ&`5kf(yc;El7LNxfSBK=R4c6%l@- zh^)`up$kzV*keT&c6`NPGyVc=e{V<#ag%Ssw6x!CpQl|~Y!37s+3bcpr-n7Jif`M_ zJ9Qq70o&=-y=82&Vlwb~=4&S$VH2sw~8 z?1#8s00aZ0(5~JF1KopQV^ghi;m_y4#5#~6*#mjCI*xlg@b!1>Jl`c*u0L!;BgE{~ zxemSWK*;H6W}~Mgsg3I0v=vET?N>IY;R_o{D{_ep@FA%DyB;)1m?yN8##!;6Px+5E z_3GJC$rzJcizn_?EFr}R_~c~CENf*eyXJ&Gf2z>+fw6pwsJ)FA)lNs2pyLj!vo7@HIt%HqilS6)B zU$QCcvkekT}EKVZUGh(UEOS{bZy%eFxVr zn`TFI61Bp}>)dMTxsvGn$Y~F&sg>TG*~x_KGsAbATZ)h3qN4cYNT_6hgymB@w+3r0 z`6D~O?rp2)rC4c4hur4+pD0SCmYpQMgKuy)t4x*zp zdqWd`o$7?}M05Bx!c||P8{A}=z-1SFPq9=m}}^GS|+K4(u7 zFZ7Jdscr~YzPHLfJdaYDG39&^LaK`=GjAqxZgDgN3s&tl!K>Cpj49#SsEX%KNy<}_ z(nUc(FTfk>zu1nQ)+|40zuk-yK7j$Y&h0AdBmB=r$|O-0;d?`-iwn8tzC(eDVGekm z6YG9$iBui6y>Eq1%&ONt5W0Z@@yBL|Xt#j3BUFpl%g=;PE?y>rA!3YML<_t z;)by$*0ypf7qJ8H6##TIs$u3(w#rwxp6px~Jp+7*ctV?)n#*%HA>^IEH<*T|410&S zYd-(sP_~eDg)n!BD)n@e5>(uzmuhcsE?KPbx+>;U3f(z4=@RQFXJb`9nNtZQ>ko&X zET4;ATpZmzK%}E@E1nG|WXoS17b=}^Bg95jUg8vn{X9u7OD+uCKP^;Nlz7*S$GWkD zl^CZkGe9|yqMUW4P>ES~qlu)Lq{0JYxd4t#EU*{z>>a!PwdR5XaKQlTosEOr+K5O) zJz6P1O{@_Sd`NE7%N~q+AgC+j#7N+{bm!H|5l9I<@V$W0-7{B(_a^m_hn9}(cTk)( zP)m*b;KbSt7G{;M%R|b2jNk7RAJ4 z!qv0$j`I$5$CHreS6PVdrXl=$o4($CVy~Ff`ZH>Aq#RabhOO#xU)~6r9lpjeafjCc z0vGPh^a_7BCq;ED1Y#OUv5ea*n-isStqLceN_2_a&U9}yDV-)^?5OROfnwZZd(Wt2 zVSFrxM0!pzrL{^K1Bh`MJmY7*IemMw#2vh-H!rXH@qG*Ja+`0Qu{)2>k*i!CybW@r zDAW8{cy*=3J<&cbVu&9rRKi{5DV{fS&(wZU=*ekG*II+0YbJiSIydAMi*Qe#;9IFM ze=4!VnqZHO*nQBu@x+j$yG$U#XO+wEsVOc~MQKw}J5e`|(+t2}KhX|ok zWm?!x&cP}Xkg}V_>~{@l?{`Gx+(Z=C_-bktT9yl?{|L|m5;2ubBlWhM4Q0{1(OeRr z28!lNHyH7MHV5u`3-K0C_D8;CKNhZ!tCR}aHhQz`D44fvWG!QT>v$$~ir4EzWt|S} zA^5N<&y(OG+wn8Ns}eRyZP{ z$!VuCptW*)nyRreQm@JCbS0h~9lpgP%del4c8V-H_nO)o{UrfVI!g1~ykbVn^9}*^ zn8Tk3(+F_=ZUX+xYW?=5e+<_@t~7D1mbhD-0{@H64omkoZ*^oGc$*VqUP10vijbb= z<%qxYdGQhO8Z$FG^a2fk{s77qoI*a&NF+D*DAlNGx-{F%1^Vpz8;XerSt~UvM`FXw zO;hhuN(aGc)wl1`?nFd!Xa-PlmWT6_ev+3ESg$G+FP%8jSYD}-^{OJLn$?085$RdNTrJFDB};`KVG9=RSahzga{W1T{+gksg&h> znEe`Idg6FBu-St za>F#S;8djAle3H0cVooemquK1W=On@*adDptr#B-RujMPMv&IZEo(v{Gq1Cu=F&WC zc%jOhwvdTv%A6_dX_r47dg4W3muyz8uYR<+~16$_M-xjlf@Bl2<1Z9Mko5@y`sTqFBGcaOQEM_D8Nt`3!CO2EBNmt zQcSk~c{GJzF8?wxy$W!3s`on_0B8zN!c@S2&G@bcUY&jY2`t9kjQ{^h*j1dX6QVyj zJ(%*t;QW_}{gJP$C|3s*e^4G{y59ew{O`EpYVcK;{U;ck;NLy>RfMZP^A7?9lPAp2 z|39wtD#BF@_Xk0M=r_Xu8M>>XS9|+Ep}LqkE#}9scKTNVuAUr!07NmO`gaQc^&Ghh xcvY7F0CH0Oi%4G$zp6if!UHjX1N@)H|3{4sb@A{ozd?$5Qe#SngZA>>{{X;;djbFe literal 4969 zcmZ`-1yodP*B(-77)pkc6k+I)PDx=1ks3j22&F+8Iz+lbK)M?SbVN!T6p+qKcZjr% zG~cM}|Gby~e&?+7o^{qb&pvxUXV<&69$;Zp004lS0FB5!-5YWm<^kxZYV<{lzAPOr zv|JpWTzSo$oOs~&b{f%A1epA!_pf@KKYoyFl{c3i)Tw$);zi3FYpL*7-nU7tHO7I# z@s`p&zBT7k2{CuDZp<=sJPCl3I=lvAT;xEPY*S$20gzaa>yI2zQ8g_aWhllgc(bN) zNgS<|HA-Zz0pDP&^-eo^ZVjSO-ur<;8hq|69lha+anp~Iw z0NLNwwQzKS{nWQFep;=Ak2Ls+qRTua?dbjnN@-tB&Pi^chxJ`U>zb^<@XmBv*DOyi z$pXyk()*A|y(myiddc-W1`CZkbprikAZPhsnH)F$^HDaiQrEWCwQ*&xyE4nffx3(? zAuVZZAaML>JnNShPVoJ4cOyJX?f4kwMm+*)@2;E{BqrTpZXUM%EaujF*diYheL99< zB;V9`W7&_S4~{hLHUSdsMV3jUqR~gSoOO)~1df_9ys+7x+WW(Izcj~_x!tU_D(Xg* zHJtTfZF#hNjLqn7j|ICrdvFD!0chaSN@;;Y#Tvcr$i}ycM*edYm_}fmqeVQaTc=N1ZQF(ns)Z=i6$9y9=H1Pm#`WNr?c*6(Y zxdnXzRx&w0nj*wr3x0vR$$KMPK)a|omDM2Eeg-H_Rq@!hUDUk(qMXFF&)iJGd6D=dwAs5$b(XMQWq zCLKQtsyXt^eRd=|tgB)WPSmuo8PZ-E)>Ss>wq?tH=c4ji{w9@;ywdWPCy?^}s_e*Z zfPvGC@h`(zNhWtspC^L;qHma$d%S0`BNj$>+0}_XZaYkqB&6h$5sO0%xYseHuH}u8 zs|jCdiywX*J3esty%y+Z>gRwXl9RKie{lqV0yXfrN9{ zjg_pTgb5A#ey2t--u@X2)GXwVg9N#YIGwe;_W%kNSHLDPuzP~P1(CF7)Jv)|9vE23 zjC5gXm_-@*pfr^oSSCtZ-Z4)kDzPeNedexTArpPCN+duob2B42<*61OWE~(7~6L& zMQ-eU8-k!#B?RsB5Sw91&r6(>bYXnDI9hs4Yua~8OhL8{dkhHH=+uffGA?;(a~ctl z%X_MT8t0NAQzKy45F9d$Bs(`O7fz5i{rs&j?s@-s95GqE$vyPD{SmQb%?!b!YN^|~ z&IZ^QnI`S3ts8;8Ru{Ft!DE7U;OHO9qtt3EBT1-g4D`ad8k)M-R(2fNBId{AGf=Qw_gyMy_`^LGAM4hU2M-=**KCtXCs-AS>mmTwm}(wY0)r+={qAyN&wS*T(k1iX`Dtv!$E z+2icM+pvVP;q~f{!0v?uGv#R=HXk0Ty`ES_)N#~GLRU5sId0P6ffLCm>&QL=L4^-4 z55CqUHoa#TRr_B3#R{a?3s1M+H6zEt;>4mVAZXU=m#D_52~7;N+BsB1DA#spR)+bAXEfE z6^lgbLbcpOnuks6T&f+9M4J@_YkKHR?~wEr*37ERf+W{-;EV2eNnT^rw$1cD(dMTon#np zikK{~>2Rs21y;0n;$v~iG5Qj_@9$v-S{Bmzr<@XeJRq#QOW9u`E#{{Pei;lg8D;}g zgjW&bFj2cK@9ri%b&s4l1qav%=?YkmQTHf>wMq%PC`{_I9ac`U6TF&U((P*iHL;sM zX!mr;;J$M;Z89o_q?S@T%zLxI@GitP&9k;mvHZ$loZ&=ZUd(A)d=ATGjU23y@7wAV z9_~+8X5Lhf&^%#Fmq2>$2i#6OAD_-g%`OmnkCqt~SaOlcJh{ca3H>hgB!6SqKm^Z;eS&jlD7iE zmz=rWC8}!ch+f=|XvX+w0V#Lr1BPM%0QOh_z^&f}#NE;5nX3&9=H|-#>-MXP_&l6; zXyYTic+JTu+hea#Tp^TFxNQc+wD*EQla*F)Zl1`1En+2Q%O zexT^-%D-QplsHw6-Th5PKYK}VN~4@@FoEIci-%B5_aL1@_5C!-XN@2P7;*H-9y5k}etq&Pi^)!`fs%}PpY%tNWGOOsM z9;lA&4#XCtD5t6&>*IQS_QoAIi{>q>Ks{dMu3(q|BXVk6vG!@@IrsKp?ZcfFzQvmY zxf^!J=~nuqU;Rd_t?;g9(2 z$|==+kan*Kc-9ZFOfch-BRmJZs;Os8I=@!_7k0FiOVEuG6#)Q1^!x9*x_Q{aTz?k+ zS-t7-X+F{m%0-gmgp}0UjsOu-4kXO%M-D;_{0Y<01t!LNc9aoExBfP@VB3ORgrRTf zjbJLxqVdsAd50@YkVTQ+W|KN-yTI!8S zS{T?hHk>0t%=2^tFwlSX-=0=*)~1#XgW@LjCYB_EMdWAwrd&q?4wTZ3)!lJ!G)1GJc@&2BZ1ngzI>@? zbUcwz_QMJ;@(I~ZUNeRa(!(K2Y=(~s9X7ISUOA(JJEw-4n*&T25Bub^n)N2lgU1?E zoYsa2&Dt=R=amSm9ou*=?}E2hhH81S(v{DS5Ury7mWte(zCNXQd^k_jLy3Loo_{?z z?0)H>D0;?7P^o|3W)W3=KsoA!|6_qv#7x;NVwEd71=^+QeH@zv2_{i zpV9U&NMP=&vo!Vpdc1#l=5Kd#Ct54rzmu$kgPs#BoZKL>|H>_s z%_GxayN)~WbNmWe6=K>M8+D~4?hL`mI#!Kf?mb8^-LN9>GgTPl9@yt)i0UDtNhIdH zFs|5i7$HBnq|=h}ag;Ak7*0^hNURiQ-mhoM<5-$4bc>aH9*{~u8nd;a-YI4nY+T@y z=fD?5f%9qs++{5^5cYNDIO<18J)5B(g)??F9M@2Z2~KMp)B|wMjs+j?wub!B_Vmy! zn79g>xSo-c!yD*Y(7sHL_G`3T1D?3R>|A+yey$nubq?)(gtC5}gPD9c2AM(yVf$3r zCm4FUB$Gg#aK(&f#8iQEECG#&lq7J5!B%!#G`}XIS@dP(5aUNZ79j_;Qg6;%^e+|{S4nI?IE>xS<>^@PK>5H zLxBaG5sKWQR9rsItZ7!8v-XqI$SH(-0x9@-X=JV+-2P?eK;0W08r_}j&a_rYW=m}g z>cwr0x;HAXRmhIgb6ZsaL2;p-xU!889N*bxdPcOCCnbz0jcmQbR+GcWN3}34()peE zYCvvP%Z40eq%Cf@urgqIHp;$E`@&kCb4b8%!V&tsA-=nA=8KX!3E1E=!2M&lCbR>C z4C-wl9G~V<7%1$LxM)83;p;xOr=&y1Ah|e(2{gdEu(H!>116;CNolcRmTD13aH$!k zJ7Lh4+S9KPTdj;cK}%TODw4`-KvG8oCJu0ODCJKtE-NdA2dM28f-W=)Ig9yaj|#=L zow&y~J(+*jHCBeS+zK?Q$7q5Tm=Ygl_=n-8yj%(km4M&0D+yA_#b#*VZ@5NnZ3>A_S@}-WZ6)- zJ7m`!vMN2p?Xm3rwhPFIBExz5m-#fGQkO3o2IDpke|$0R= z`K+zC=*3&nJrce~U%#FEBlu5D(4BE;BJ@KiZ?9QBc29r-~hUK|J?mS6&xMhU=D6ibUmD4uEsyT zs3JiF(#}VUwhjhm;{h9F7t)#%7&~EDXoAWv6ReGfrv)KXYS}=_R~qH;xyFIainm^vv)|B$Jrxd1?)_+WQ13>5tykms+jl@KO%1#H}$wq>Ez- zeA9zgV^8Bvi@Bj>RNI%QYBYJmYQTxk)-A@wulI}VTnA09nRyJf!pnjlpU^vf^^%ZP zQ&Omo3o|LOc)d{}D{j={IknV7qc9D^5ikC@oCLDKEUCu2ZM`+*#;?Ji$rm}$@Yi{( z75?WQ4SX`OYHYtx87$qik1SbKEDI5Erng7m&g3A_mrZef%NK;U_f`VT4b}#Zy}GEc zX#4lc+IE^{svE~4AiuSUMHv%6G?N&Z6j=XzEP|%<*Wr)m_x}$`uA{F{DSl%C0Dla` z-_id!v$zhwzK#C_??Zq7UpDpE1zd0A{wpAW_~tJG|Ju}DmvX(G_^%X|Uk{5hHe{0E{;phW-x diff --git a/databases.ipynb b/databases.ipynb index b295748..661cfd1 100644 --- a/databases.ipynb +++ b/databases.ipynb @@ -417,8 +417,8 @@ "outputs": [], "source": [ "df = pl.read_database(\n", - " query=sql_join, # your SQL query (string)\n", - " connection=con # your connection object (SQLAlchemy, psycopg2 cursor, etc.)\n", + " query=sql_join, # your SQL query (string)\n", + " connection=con, # your connection object (SQLAlchemy, psycopg2 cursor, etc.)\n", ")" ] }, diff --git a/rectangling.ipynb b/rectangling.ipynb index 09d3402..199c215 100644 --- a/rectangling.ipynb +++ b/rectangling.ipynb @@ -576,8 +576,7 @@ " {\"name\": {\"given\": \"Mark\", \"family\": \"Regner\"}},\n", " {\"id\": 2, \"name\": \"Faye Raker\"},\n", "]\n", - "pl.json_normalize(data)\n", - "\n" + "pl.json_normalize(data)" ] }, { diff --git a/spreadsheets.ipynb b/spreadsheets.ipynb index 8053af7..ee7f5bf 100644 --- a/spreadsheets.ipynb +++ b/spreadsheets.ipynb @@ -1,464 +1,820 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "95f0a171", - "metadata": {}, - "source": [ - "# Spreadsheets {#sec-spreadsheets}\n", - "\n", - "## Introduction\n", - "\n", - "This chapter will show you how to work with spreadsheets, for example Microsoft Excel files, in Python. We already saw how to import csv (and tsv) files in @sec-data-import. In this chapter we will introduce you to tools for working with data in Excel spreadsheets and Google Sheets.\n", - "\n", - "If you or your collaborators are using spreadsheets for organising data that will be ingested by an analytical tool like Python, we recommend reading the paper \"Data Organization in Spreadsheets\" by Karl Broman and Kara Woo {cite}`broman2018data`. The best practices presented in this paper will save you much headache down the line when you import the data from a spreadsheet into Python to analyse and visualise. (For spreadsheets that are meant to be read by humans, we recommend the [good practice tables](https://github.com/best-practice-and-impact/gptables) package.)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "51a55374", - "metadata": { - "tags": [ - "remove-cell" - ] - }, - "outputs": [], - "source": [ - "# remove cell\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib_inline.backend_inline\n", - "\n", - "# Plot settings\n", - "plt.style.use(\"https://github.com/aeturrell/python4DS/raw/main/plot_style.txt\")\n", - "matplotlib_inline.backend_inline.set_matplotlib_formats(\"svg\")" - ] - }, - { - "cell_type": "markdown", - "id": "17575f3a", - "metadata": {}, - "source": [ - "### Prerequisites\n", - "\n", - "You will need the **polars** package for this chapter. Install **fastexcel** so `read_excel()` can use the default (fast) engine, **xlsxwriter** for `write_excel()`, and **openpyxl** if you want to use the **openpyxl** engine explicitly (`uv add fastexcel xlsxwriter openpyxl`)." - ] - }, - { - "cell_type": "markdown", - "id": "ff99055a", - "metadata": {}, - "source": [ - "## Reading Excel (and Similar) Files\n", - "\n", - "**polars** can read in xls, xlsx, xlsm, xlsb, odf, ods, and odt files from your local filesystem or from a URL. It also supports an option to read a single sheet or a list of sheets. The default reader uses **fastexcel** (Rust-backed, via the **fastexcel** Python package); you can also select other engines such as **openpyxl** when you need engine-specific options.\n", - "\n", - "To show how this works, we'll work with an example spreadsheet called \"students.xlsx\". The figure below shows what the spreadsheet looks like.\n", - "\n", - "![A look at the students spreadsheet in Excel. The spreadsheet contains information on 6 students, their ID, full name, favourite food, meal plan, and age.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-students.png)" - ] - }, - { - "cell_type": "markdown", - "id": "29f2f4e0", - "metadata": {}, - "source": [ - "The first argument to `pl.read_excel()` is the path to the file to read. If you have downloaded the [file]() onto your computer and put it in a subfolder called \"data\" then you would want to use the path \"data/students.xlsx\" but we can also load it directly from the URL." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bf00d4c6", - "metadata": {}, - "outputs": [], - "source": [ - "import polars as pl\n", - "\n", - "students = pl.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/students.xlsx\",\n", - ")\n", - "students" - ] - }, - { - "cell_type": "markdown", - "id": "03d6ba9c", - "metadata": {}, - "source": [ - "We have six students in the data and five variables on each student. However there are a few things we might want to address in this dataset:\n", - "\n", - "- The column names are all over the place. You can rename them to follow a consistent format; we recommend `snake_case`. If you want to replace **every** column name in order, assigning to `students.columns` is clear and short. If you only rename some columns, use `.rename({\"Old Name\": \"new_name\", ...})` with the exact strings from the sheet.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ecce26ea", - "metadata": {}, - "outputs": [], - "source": [ - "students.columns = [\n", - " \"student_id\",\n", - " \"full_name\",\n", - " \"favourite_food\",\n", - " \"meal_plan\",\n", - " \"age\",\n", - "]\n", - "students" - ] - }, - { - "cell_type": "markdown", - "id": "bb07ad4f", - "metadata": {}, - "source": [ - "\n", - "- `age` may be inferred as strings (for example **Utf8**) when the column mixes numeric values and text, but we want it numeric. Just like with `read_csv()`, you can supply a `schema_overrides` argument to `read_excel()` and specify Polars data types for the columns you read in (for example `pl.Int64`, `pl.Utf8`, `pl.Boolean`, `pl.Datetime`, and more). That still will not fix a value like `\"five\"` until we map it to a number first.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "22e45cc2", - "metadata": {}, - "outputs": [], - "source": [ - "students = pl.read_excel(\"data/students.xlsx\")\n", - "students.columns = [\n", - " \"student_id\",\n", - " \"full_name\",\n", - " \"favourite_food\",\n", - " \"meal_plan\",\n", - " \"age\",\n", - "]\n", - "students = students.with_columns(pl.col(\"age\").replace({\"five\": 5}))\n", - "students" - ] - }, - { - "cell_type": "markdown", - "id": "c8a07159", - "metadata": {}, - "source": [ - "Okay, now we can apply the data types." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f67490d3", - "metadata": {}, - "outputs": [], - "source": [ - "students = students.with_columns(\n", - " [\n", - " pl.col(\"student_id\").cast(pl.Int64),\n", - " pl.col(\"full_name\").cast(pl.Utf8),\n", - " pl.col(\"favourite_food\").cast(pl.Utf8),\n", - " pl.col(\"meal_plan\").cast(pl.Categorical),\n", - " pl.col(\"age\").cast(pl.Int64),\n", - " ]\n", - ")\n", - "students.schema" - ] - }, - { - "cell_type": "markdown", - "id": "362ff5a5", - "metadata": {}, - "source": [ - "It took multiple steps and trial-and-error to load the data in exactly the format we want, and this is not unexpected. Data science is an iterative process. There is no way to know exactly what the data will look like until you load it and take a look at it. The general pattern we used is load the data, take a peek, make adjustments to your code, load it again, and repeat until you're happy with the result." - ] - }, - { - "cell_type": "markdown", - "id": "8b00c2a4", - "metadata": {}, - "source": [ - "### Reading Individual Sheets\n", - "\n", - "An important feature that distinguishes spreadsheets from flat files is the notion of multiple sheets. The figure below shows an Excel spreadsheet with multiple sheets. The data come from the **palmerpenguins** dataset {cite}`horst2020palmerpenguins`. Each sheet contains information on penguins from a different island where data were collected.\n", - "\n", - "![A look at the penguins spreadsheet in Excel. The spreadsheet contains has three sheets: Torgersen Island, Biscoe Island, and Dream Island.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-penguins-islands.png)\n", - "\n", - "You can read a single sheet using the following command (so as not to show the whole file, we'll use `.head()` to just show the first 5 rows):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e37f9e3d", - "metadata": {}, - "outputs": [], - "source": [ - "pl.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/penguins.xlsx\",\n", - " sheet_name=\"Torgersen Island\",\n", - ").head()" - ] - }, - { - "cell_type": "markdown", - "id": "641f6831", - "metadata": {}, - "source": [ - "Now this relies on us knowing the names of the sheets in advance. There will be situations where you want to read in data without peeking into the Excel spreadsheet. To read all sheets in Polars, use `sheet_id=0` (or `sheet_name=None`, which also works in recent versions of Polars). The object that’s created is a dictionary where the keys are the sheet names and the values are Polars DataFrames. To access a specific sheet, you can convert the keys() or values() to a list and then index into it, ie `list(dictionary.keys())[]` .\n", - "\n", - "To give a sense of how this works, let's first print all of the retrieved keys:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0f3a3b78", - "metadata": {}, - "outputs": [], - "source": [ - "penguins_dict = pl.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/penguins.xlsx\",\n", - " sheet_id=0,\n", - ")\n", - "print([x for x in penguins_dict.keys()])" - ] - }, - { - "cell_type": "markdown", - "id": "076f1ebe", - "metadata": {}, - "source": [ - "Now let's show the second entry data frame" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15495426", - "metadata": {}, - "outputs": [], - "source": [ - "print(list(penguins_dict.keys())[1])\n", - "list(penguins_dict.values())[1].head()" - ] - }, - { - "cell_type": "markdown", - "id": "536ab4bb", - "metadata": {}, - "source": [ - "What we really want is these three *consistent* datasets to be in the *same* single data frame. For this, we can use the `pl.concat()` function. This concatenates any given iterable of data frames." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "151ba846", - "metadata": {}, - "outputs": [], - "source": [ - "penguins = pl.concat(penguins_dict.values())\n", - "penguins" - ] - }, - { - "cell_type": "markdown", - "id": "aa458187", - "metadata": {}, - "source": [ - "### Reading Part of a Sheet\n", - "\n", - "Since many use Excel spreadsheets for presentation as well as for data storage, it's quite common to find cell entries in a spreadsheet that are not part of the data you want to read in.\n", - "\n", - "The figure below shows such a spreadsheet: in the middle of the sheet is what looks like a data frame but there is extraneous text in cells above and below the data.\n", - "\n", - "![A look at the deaths spreadsheet in Excel. The spreadsheet has four rows on top that contain non-data information; the text 'For the same of consistency in the data layout, which is really a beautiful thing, I will keep making notes up here.' is spread across cells in these top four rows. Then, there is a data frame that includes information on deaths of 10 famous people, including their names, professions, ages, whether they have kids or not, date of birth and death. At the bottom, there are four more rows of non-data information; the text 'This has been really fun, but we're signing off now!' is spread across cells in these bottom four rows.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-deaths.png)\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "id": "61505c86", - "metadata": {}, - "source": [ - "This spreadsheet can be downloaded from [here](https://github.com/aeturrell/python4DS/tree/main/data) or you can load it directly from a URL. If you want to load it from your own computer's disk, you'll need to save it in a sub-folder called \"data\" first.\n", - "\n", - "\n", - "The top three rows and the bottom four rows are not part of the data frame. We could skip the top three rows by passing `read_options` to `read_excel()`. Note that we set `skip_rows=4` since the fourth row contains column names, not the data.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "29987b9d", - "metadata": {}, - "outputs": [], - "source": [ - "pl.read_excel(\n", - " \"data/deaths.xlsx\",\n", - " read_options={\"skip_rows\": 4},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "a1a8c3ca", - "metadata": {}, - "source": [ - "We could also set `n_rows` inside `read_options` to omit the extraneous rows at the bottom (another option would be to skip a set number of rows at the end using `skip_footer` in `read_options`, depending on the engine)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9d7a3db3", - "metadata": {}, - "outputs": [], - "source": [ - "pl.read_excel(\n", - " \"data/deaths.xlsx\",\n", - " read_options={\"skip_rows\": 4, \"n_rows\": 10},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "2d277c3a", - "metadata": {}, - "source": [ - "## Data Types\n", - "\n", - "In CSV files, all values are strings. This is not particularly true to the data, but it is simple: everything is a string.\n", - "\n", - "The underlying data in Excel spreadsheets is more complex. A cell can be one of five things:\n", - "\n", - "- A logical, like TRUE / FALSE\n", - "\n", - "- A number, like \"10\" or \"10.5\"\n", - "\n", - "- A date, which can also include time like \"11/1/21\" or \"11/1/21 3:00 PM\"\n", - "\n", - "- A string, like \"ten\"\n", - "\n", - "- A currency, which allows numeric values in a limited range and four decimal digits of fixed precision\n", - "\n", - "When working with spreadsheet data, it's important to keep in mind that how the underlying data is stored can be very different than what you see in the cell. For example, Excel has no notion of an integer. All numbers are stored as floating points (real number), but you can choose to display the data with a customizable number of decimal points. Similarly, dates are actually stored as numbers, specifically the number of seconds since January 1, 1970. You can customize how you display the date by applying formatting in Excel. Confusingly, it's also possible to have something that looks like a number but is actually a string (e.g. type `'10` into a cell in Excel).\n", - "\n", - "These differences between how the underlying data are stored vs. how they're displayed can cause surprises when the data are loaded into analytical tools such as **polars**. By default, **polars** will guess the data type in a given column.\n", - "A recommended workflow is to let **polars** guess the column types initially, inspect them, and then change any data types that you want to." - ] - }, - { - "cell_type": "markdown", - "id": "0d73bbad", - "metadata": {}, - "source": [ - "## Writing to Excel\n", - "\n", - "Let's create a small data frame that we can then write out. Note that `item` is a category and `quantity` is an integer." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15963e18", - "metadata": {}, - "outputs": [], - "source": [ - "bake_sale = pl.DataFrame(\n", - " {\n", - " \"item\": pl.Series([\"brownie\", \"cupcake\", \"cookie\"], dtype=pl.Categorical),\n", - " \"quantity\": [10, 5, 8],\n", - " }\n", - ")\n", - "bake_sale" - ] - }, - { - "cell_type": "markdown", - "id": "345bca3d", - "metadata": {}, - "source": [ - "You can write data back to disk as an Excel file using the `.write_excel()` method. Polars does not use a row index like pandas, so only the columns in the DataFrame are written by default." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1fc17141", - "metadata": {}, - "outputs": [], - "source": [ - "bake_sale.write_excel(\"data/bake_sale.xlsx\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "f892c2e7", - "metadata": {}, - "source": [ - "The figure below shows what the data looks like in Excel.\n", - "\n", - "![Bake sale data frame created earlier in Excel.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-bake-sale.png)" - ] - }, - { - "cell_type": "markdown", - "id": "8d555c84", - "metadata": {}, - "source": [ - "Just like reading from a CSV, information on data type is lost when we read the data back in—you can see this if you read the data back in and check the `schema` for the data types. Although we kept `Int64` because **polars** recognised that the second column was of integer type, we lost the categorical data type for \"item\". This data type loss makes Excel files unreliable for caching interim results." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "27e128f9", - "metadata": {}, - "outputs": [], - "source": [ - "pl.read_excel(\"data/bake_sale.xlsx\").schema" - ] - }, - { - "cell_type": "markdown", - "id": "215634bf", - "metadata": {}, - "source": [ - "### Formatted Output\n", - "\n", - "If you need more formatting options and more control over how you write spreadsheets, check out the documentation for [openpyxl](https://openpyxl.readthedocs.io/) which can do pretty much everything you imagine. Generally, releasing data in spreadsheets is not the best option: but if you do want to release data in spreadsheets according to best practice, then check out [gptables](https://gptables.readthedocs.io/)." - ] + "cells": [ + { + "cell_type": "markdown", + "id": "95f0a171", + "metadata": {}, + "source": [ + "# Spreadsheets {#sec-spreadsheets}\n", + "\n", + "## Introduction\n", + "\n", + "This chapter will show you how to work with spreadsheets, for example Microsoft Excel files, in Python. We already saw how to import csv (and tsv) files in @sec-data-import. In this chapter we will introduce you to tools for working with data in Excel spreadsheets and Google Sheets.\n", + "\n", + "If you or your collaborators are using spreadsheets for organising data that will be ingested by an analytical tool like Python, we recommend reading the paper \"Data Organization in Spreadsheets\" by Karl Broman and Kara Woo {cite}`broman2018data`. The best practices presented in this paper will save you much headache down the line when you import the data from a spreadsheet into Python to analyse and visualise. (For spreadsheets that are meant to be read by humans, we recommend the [good practice tables](https://github.com/best-practice-and-impact/gptables) package.)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "51a55374", + "metadata": { + "tags": [ + "remove-cell" + ] + }, + "outputs": [], + "source": [ + "# remove cell\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib_inline.backend_inline\n", + "\n", + "# Plot settings\n", + "plt.style.use(\"https://github.com/aeturrell/python4DS/raw/main/plot_style.txt\")\n", + "matplotlib_inline.backend_inline.set_matplotlib_formats(\"svg\")" + ] + }, + { + "cell_type": "markdown", + "id": "17575f3a", + "metadata": {}, + "source": [ + "### Prerequisites\n", + "\n", + "You will need the **polars** package for this chapter. Install **fastexcel** so `read_excel()` can use the default (fast) engine, **xlsxwriter** for `write_excel()`, and **openpyxl** if you want to use the **openpyxl** engine explicitly (`uv add fastexcel xlsxwriter openpyxl`).\n" + ] + }, + { + "cell_type": "markdown", + "id": "ff99055a", + "metadata": {}, + "source": [ + "## Reading Excel (and Similar) Files\n", + "\n", + "**polars** can read in xls, xlsx, xlsm, xlsb, odf, ods, and odt files from your local filesystem or from a URL. It also supports an option to read a single sheet or a list of sheets. The default reader uses **fastexcel** (Rust-backed, via the **fastexcel** Python package); you can also select other engines such as **openpyxl** when you need engine-specific options.\n", + "\n", + "To show how this works, we'll work with an example spreadsheet called \"students.xlsx\". The figure below shows what the spreadsheet looks like.\n", + "\n", + "![A look at the students spreadsheet in Excel. The spreadsheet contains information on 6 students, their ID, full name, favourite food, meal plan, and age.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-students.png)\n" + ] + }, + { + "cell_type": "markdown", + "id": "29f2f4e0", + "metadata": {}, + "source": [ + "The first argument to `pl.read_excel()` is the path to the file to read. If you have downloaded the [file]() onto your computer and put it in a subfolder called \"data\" then you would want to use the path \"data/students.xlsx\" but we can also load it directly from the URL.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "bf00d4c6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 5)
Student IDFull Namefavourite.foodmealPlanAGE
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only"null
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""five"
6"Güvenç Attila""Ice cream""Lunch only""6"
" + ], + "text/plain": [ + "shape: (6, 5)\n", + "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", + "│ Student ID ┆ Full Name ┆ favourite.food ┆ mealPlan ┆ AGE │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", + "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", + "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", + "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", + "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", + "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ null │\n", + "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ five │\n", + "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", + "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" } - ], - "metadata": { - "interpreter": { - "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" - }, - "jupytext": { - "cell_metadata_filter": "-all", - "encoding": "# -*- coding: utf-8 -*-", - "formats": "md:myst", - "main_language": "python" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.12" - }, - "toc-showtags": true + ], + "source": [ + "import polars as pl\n", + "\n", + "students = pl.read_excel(\n", + " \"data/students.xlsx\",\n", + ")\n", + "students" + ] + }, + { + "cell_type": "markdown", + "id": "03d6ba9c", + "metadata": {}, + "source": [ + "We have six students in the data and five variables on each student. However there are a few things we might want to address in this dataset:\n", + "\n", + "- The column names are all over the place. You can rename them to follow a consistent format; we recommend `snake_case`. If you want to replace **every** column name in order, assigning to `students.columns` is clear and short. If you only rename some columns, use `.rename({\"Old Name\": \"new_name\", ...})` with the exact strings from the sheet.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "ecce26ea", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 5)
student_idfull_namefavourite_foodmeal_planage
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only"null
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""five"
6"Güvenç Attila""Ice cream""Lunch only""6"
" + ], + "text/plain": [ + "shape: (6, 5)\n", + "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", + "│ student_id ┆ full_name ┆ favourite_food ┆ meal_plan ┆ age │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", + "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", + "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", + "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", + "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", + "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ null │\n", + "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ five │\n", + "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", + "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "students.columns = [\n", + " \"student_id\",\n", + " \"full_name\",\n", + " \"favourite_food\",\n", + " \"meal_plan\",\n", + " \"age\",\n", + "]\n", + "students" + ] + }, + { + "cell_type": "markdown", + "id": "bb07ad4f", + "metadata": {}, + "source": [ + "- `age` may be inferred as strings (for example **Utf8**) when the column mixes numeric values and text, but we want it numeric. Just like with `read_csv()`, you can supply a `schema_overrides` argument to `read_excel()` and specify Polars data types for the columns you read in (for example `pl.Int64`, `pl.Utf8`, `pl.Boolean`, `pl.Datetime`, and more). That still will not fix a value like `\"five\"` until we map it to a number first.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "22e45cc2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 5)
student_idfull_namefavourite_foodmeal_planage
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only"null
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""5"
6"Güvenç Attila""Ice cream""Lunch only""6"
" + ], + "text/plain": [ + "shape: (6, 5)\n", + "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", + "│ student_id ┆ full_name ┆ favourite_food ┆ meal_plan ┆ age │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", + "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", + "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", + "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", + "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", + "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ null │\n", + "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ 5 │\n", + "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", + "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "students = pl.read_excel(\"data/students.xlsx\")\n", + "students.columns = [\n", + " \"student_id\",\n", + " \"full_name\",\n", + " \"favourite_food\",\n", + " \"meal_plan\",\n", + " \"age\",\n", + "]\n", + "students = students.with_columns(pl.col(\"age\").replace({\"five\": 5}))\n", + "students" + ] + }, + { + "cell_type": "markdown", + "id": "c8a07159", + "metadata": {}, + "source": [ + "Okay, now we can apply the data types.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "f67490d3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Schema([('student_id', Int64),\n", + " ('full_name', String),\n", + " ('favourite_food', String),\n", + " ('meal_plan', Categorical(ordering='physical')),\n", + " ('age', Int64)])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "students = students.with_columns(\n", + " [\n", + " pl.col(\"student_id\").cast(pl.Int64),\n", + " pl.col(\"full_name\").cast(pl.Utf8),\n", + " pl.col(\"favourite_food\").cast(pl.Utf8),\n", + " pl.col(\"meal_plan\").cast(pl.Categorical),\n", + " pl.col(\"age\").cast(pl.Int64),\n", + " ]\n", + ")\n", + "students.schema" + ] + }, + { + "cell_type": "markdown", + "id": "362ff5a5", + "metadata": {}, + "source": [ + "It took multiple steps and trial-and-error to load the data in exactly the format we want, and this is not unexpected. Data science is an iterative process. There is no way to know exactly what the data will look like until you load it and take a look at it. The general pattern we used is load the data, take a peek, make adjustments to your code, load it again, and repeat until you're happy with the result.\n" + ] + }, + { + "cell_type": "markdown", + "id": "8b00c2a4", + "metadata": {}, + "source": [ + "### Reading Individual Sheets\n", + "\n", + "An important feature that distinguishes spreadsheets from flat files is the notion of multiple sheets. The figure below shows an Excel spreadsheet with multiple sheets. The data come from the **palmerpenguins** dataset {cite}`horst2020palmerpenguins`. Each sheet contains information on penguins from a different island where data were collected.\n", + "\n", + "![A look at the penguins spreadsheet in Excel. The spreadsheet contains has three sheets: Torgersen Island, Biscoe Island, and Dream Island.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-penguins-islands.png)\n", + "\n", + "You can read a single sheet using the following command (so as not to show the whole file, we'll use `.head()` to just show the first 5 rows):\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e37f9e3d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (5, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64i64i64stri64
"Adelie""Torgersen"39.118.71813750"male"2007
"Adelie""Torgersen"39.517.41863800"female"2007
"Adelie""Torgersen"40.318.01953250"female"2007
"Adelie""Torgersen"nullnullnullnull"NA"2007
"Adelie""Torgersen"36.719.31933450"female"2007
" + ], + "text/plain": [ + "shape: (5, 8)\n", + "┌─────────┬───────────┬───────────────┬───────────────┬──────────────┬─────────────┬────────┬──────┐\n", + "│ species ┆ island ┆ bill_length_m ┆ bill_depth_mm ┆ flipper_leng ┆ body_mass_g ┆ sex ┆ year │\n", + "│ --- ┆ --- ┆ m ┆ --- ┆ th_mm ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ --- ┆ f64 ┆ --- ┆ i64 ┆ str ┆ i64 │\n", + "│ ┆ ┆ f64 ┆ ┆ i64 ┆ ┆ ┆ │\n", + "╞═════════╪═══════════╪═══════════════╪═══════════════╪══════════════╪═════════════╪════════╪══════╡\n", + "│ Adelie ┆ Torgersen ┆ 39.1 ┆ 18.7 ┆ 181 ┆ 3750 ┆ male ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 39.5 ┆ 17.4 ┆ 186 ┆ 3800 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 40.3 ┆ 18.0 ┆ 195 ┆ 3250 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ null ┆ null ┆ null ┆ null ┆ NA ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 36.7 ┆ 19.3 ┆ 193 ┆ 3450 ┆ female ┆ 2007 │\n", + "└─────────┴───────────┴───────────────┴───────────────┴──────────────┴─────────────┴────────┴──────┘" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pl.read_excel(\n", + " \"data/penguins.xlsx\",\n", + " sheet_name=\"Torgersen Island\",\n", + ").head()" + ] + }, + { + "cell_type": "markdown", + "id": "641f6831", + "metadata": {}, + "source": [ + "Now this relies on us knowing the names of the sheets in advance. There will be situations where you want to read in data without peeking into the Excel spreadsheet. To read all sheets in Polars, use `sheet_id=0` (or `sheet_name=None`, which also works in recent versions of Polars). The object that’s created is a dictionary where the keys are the sheet names and the values are Polars DataFrames. To access a specific sheet, you can convert the keys() or values() to a list and then index into it, ie `list(dictionary.keys())[]` .\n", + "\n", + "To give a sense of how this works, let's first print all of the retrieved keys:\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0f3a3b78", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['Torgersen Island', 'Biscoe Island', 'Dream Island']\n" + ] + } + ], + "source": [ + "penguins_dict = pl.read_excel(\n", + " \"data/penguins.xlsx\",\n", + " sheet_id=0,\n", + ")\n", + "print([x for x in penguins_dict.keys()])" + ] + }, + { + "cell_type": "markdown", + "id": "076f1ebe", + "metadata": {}, + "source": [ + "Now let's show the second entry data frame\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "15495426", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Biscoe Island\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "shape: (5, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64i64i64stri64
"Adelie""Biscoe"37.818.31743400"female"2007
"Adelie""Biscoe"37.718.71803600"male"2007
"Adelie""Biscoe"35.919.21893800"female"2007
"Adelie""Biscoe"38.218.11853950"male"2007
"Adelie""Biscoe"38.817.21803800"male"2007
" + ], + "text/plain": [ + "shape: (5, 8)\n", + "┌─────────┬────────┬────────────────┬───────────────┬────────────────┬─────────────┬────────┬──────┐\n", + "│ species ┆ island ┆ bill_length_mm ┆ bill_depth_mm ┆ flipper_length ┆ body_mass_g ┆ sex ┆ year │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ _mm ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ f64 ┆ f64 ┆ --- ┆ i64 ┆ str ┆ i64 │\n", + "│ ┆ ┆ ┆ ┆ i64 ┆ ┆ ┆ │\n", + "╞═════════╪════════╪════════════════╪═══════════════╪════════════════╪═════════════╪════════╪══════╡\n", + "│ Adelie ┆ Biscoe ┆ 37.8 ┆ 18.3 ┆ 174 ┆ 3400 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Biscoe ┆ 37.7 ┆ 18.7 ┆ 180 ┆ 3600 ┆ male ┆ 2007 │\n", + "│ Adelie ┆ Biscoe ┆ 35.9 ┆ 19.2 ┆ 189 ┆ 3800 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Biscoe ┆ 38.2 ┆ 18.1 ┆ 185 ┆ 3950 ┆ male ┆ 2007 │\n", + "│ Adelie ┆ Biscoe ┆ 38.8 ┆ 17.2 ┆ 180 ┆ 3800 ┆ male ┆ 2007 │\n", + "└─────────┴────────┴────────────────┴───────────────┴────────────────┴─────────────┴────────┴──────┘" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(list(penguins_dict.keys())[1])\n", + "list(penguins_dict.values())[1].head()" + ] + }, + { + "cell_type": "markdown", + "id": "536ab4bb", + "metadata": {}, + "source": [ + "What we really want is these three _consistent_ datasets to be in the _same_ single data frame. For this, we can use the `pl.concat()` function. This concatenates any given iterable of data frames.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "151ba846", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (344, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64i64i64stri64
"Adelie""Torgersen"39.118.71813750"male"2007
"Adelie""Torgersen"39.517.41863800"female"2007
"Adelie""Torgersen"40.318.01953250"female"2007
"Adelie""Torgersen"nullnullnullnull"NA"2007
"Adelie""Torgersen"36.719.31933450"female"2007
……………………
"Chinstrap""Dream"55.819.82074000"male"2009
"Chinstrap""Dream"43.518.12023400"female"2009
"Chinstrap""Dream"49.618.21933775"male"2009
"Chinstrap""Dream"50.819.02104100"male"2009
"Chinstrap""Dream"50.218.71983775"female"2009
" + ], + "text/plain": [ + "shape: (344, 8)\n", + "┌───────────┬───────────┬──────────────┬──────────────┬──────────────┬─────────────┬────────┬──────┐\n", + "│ species ┆ island ┆ bill_length_ ┆ bill_depth_m ┆ flipper_leng ┆ body_mass_g ┆ sex ┆ year │\n", + "│ --- ┆ --- ┆ mm ┆ m ┆ th_mm ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ --- ┆ --- ┆ --- ┆ i64 ┆ str ┆ i64 │\n", + "│ ┆ ┆ f64 ┆ f64 ┆ i64 ┆ ┆ ┆ │\n", + "╞═══════════╪═══════════╪══════════════╪══════════════╪══════════════╪═════════════╪════════╪══════╡\n", + "│ Adelie ┆ Torgersen ┆ 39.1 ┆ 18.7 ┆ 181 ┆ 3750 ┆ male ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 39.5 ┆ 17.4 ┆ 186 ┆ 3800 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 40.3 ┆ 18.0 ┆ 195 ┆ 3250 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ null ┆ null ┆ null ┆ null ┆ NA ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 36.7 ┆ 19.3 ┆ 193 ┆ 3450 ┆ female ┆ 2007 │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ Chinstrap ┆ Dream ┆ 55.8 ┆ 19.8 ┆ 207 ┆ 4000 ┆ male ┆ 2009 │\n", + "│ Chinstrap ┆ Dream ┆ 43.5 ┆ 18.1 ┆ 202 ┆ 3400 ┆ female ┆ 2009 │\n", + "│ Chinstrap ┆ Dream ┆ 49.6 ┆ 18.2 ┆ 193 ┆ 3775 ┆ male ┆ 2009 │\n", + "│ Chinstrap ┆ Dream ┆ 50.8 ┆ 19.0 ┆ 210 ┆ 4100 ┆ male ┆ 2009 │\n", + "│ Chinstrap ┆ Dream ┆ 50.2 ┆ 18.7 ┆ 198 ┆ 3775 ┆ female ┆ 2009 │\n", + "└───────────┴───────────┴──────────────┴──────────────┴──────────────┴─────────────┴────────┴──────┘" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "penguins = pl.concat(penguins_dict.values())\n", + "penguins" + ] + }, + { + "cell_type": "markdown", + "id": "aa458187", + "metadata": {}, + "source": [ + "### Reading Part of a Sheet\n", + "\n", + "Since many use Excel spreadsheets for presentation as well as for data storage, it's quite common to find cell entries in a spreadsheet that are not part of the data you want to read in.\n", + "\n", + "The figure below shows such a spreadsheet: in the middle of the sheet is what looks like a data frame but there is extraneous text in cells above and below the data.\n", + "\n", + "![A look at the deaths spreadsheet in Excel. The spreadsheet has four rows on top that contain non-data information; the text 'For the same of consistency in the data layout, which is really a beautiful thing, I will keep making notes up here.' is spread across cells in these top four rows. Then, there is a data frame that includes information on deaths of 10 famous people, including their names, professions, ages, whether they have kids or not, date of birth and death. At the bottom, there are four more rows of non-data information; the text 'This has been really fun, but we're signing off now!' is spread across cells in these bottom four rows.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-deaths.png)\n" + ] + }, + { + "cell_type": "markdown", + "id": "61505c86", + "metadata": {}, + "source": [ + "This spreadsheet can be downloaded from [here](https://github.com/aeturrell/python4DS/tree/main/data) or you can load it directly from a URL. If you want to load it from your own computer's disk, you'll need to save it in a sub-folder called \"data\" first.\n", + "\n", + "The top three rows and the bottom four rows are not part of the data frame. We could skip the top three rows by passing `read_options` to `read_excel()`. Note that we set `skip_rows=4` since the fourth row contains column names, not the data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "29987b9d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (14, 6)
Lots of people__UNNAMED__1__UNNAMED__2__UNNAMED__3__UNNAMED__4__UNNAMED__5
strstrstrstrdatestr
"David Bowie""musician""69""true"1947-01-08"2016-01-10 00:00:00"
"Carrie Fisher""actor""60""true"1956-10-21"2016-12-27 00:00:00"
"Chuck Berry""musician""90""true"1926-10-18"2017-03-18 00:00:00"
"Bill Paxton""actor""61""true"1955-05-17"2017-02-25 00:00:00"
"Prince""musician""57""true"1958-06-07"2016-04-21 00:00:00"
………………
"George Michael""musician""53""false"1963-06-25"2016-12-25 00:00:00"
"Some"nullnullnullnullnull
null"also like to write stuff"nullnullnullnull
nullnull"at the""bottom,"nullnull
nullnullnullnullnull"too!"
" + ], + "text/plain": [ + "shape: (14, 6)\n", + "┌────────────────┬──────────────┬──────────────┬──────────────┬──────────────┬─────────────────────┐\n", + "│ Lots of people ┆ __UNNAMED__1 ┆ __UNNAMED__2 ┆ __UNNAMED__3 ┆ __UNNAMED__4 ┆ __UNNAMED__5 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ str ┆ str ┆ date ┆ str │\n", + "╞════════════════╪══════════════╪══════════════╪══════════════╪══════════════╪═════════════════════╡\n", + "│ David Bowie ┆ musician ┆ 69 ┆ true ┆ 1947-01-08 ┆ 2016-01-10 00:00:00 │\n", + "│ Carrie Fisher ┆ actor ┆ 60 ┆ true ┆ 1956-10-21 ┆ 2016-12-27 00:00:00 │\n", + "│ Chuck Berry ┆ musician ┆ 90 ┆ true ┆ 1926-10-18 ┆ 2017-03-18 00:00:00 │\n", + "│ Bill Paxton ┆ actor ┆ 61 ┆ true ┆ 1955-05-17 ┆ 2017-02-25 00:00:00 │\n", + "│ Prince ┆ musician ┆ 57 ┆ true ┆ 1958-06-07 ┆ 2016-04-21 00:00:00 │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ George Michael ┆ musician ┆ 53 ┆ false ┆ 1963-06-25 ┆ 2016-12-25 00:00:00 │\n", + "│ Some ┆ null ┆ null ┆ null ┆ null ┆ null │\n", + "│ null ┆ also like to ┆ null ┆ null ┆ null ┆ null │\n", + "│ ┆ write stuff ┆ ┆ ┆ ┆ │\n", + "│ null ┆ null ┆ at the ┆ bottom, ┆ null ┆ null │\n", + "│ null ┆ null ┆ null ┆ null ┆ null ┆ too! │\n", + "└────────────────┴──────────────┴──────────────┴──────────────┴──────────────┴─────────────────────┘" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pl.read_excel(\n", + " \"data/deaths.xlsx\",\n", + " read_options={\"skip_rows\": 4},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "a1a8c3ca", + "metadata": {}, + "source": [ + "We could also set `n_rows` inside `read_options` to omit the extraneous rows at the bottom (another option would be to skip a set number of rows at the end using `skip_footer` in `read_options`, depending on the engine).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "9d7a3db3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (10, 6)
Lots of people__UNNAMED__1__UNNAMED__2__UNNAMED__3__UNNAMED__4__UNNAMED__5
strstri64booldatedate
"David Bowie""musician"69true1947-01-082016-01-10
"Carrie Fisher""actor"60true1956-10-212016-12-27
"Chuck Berry""musician"90true1926-10-182017-03-18
"Bill Paxton""actor"61true1955-05-172017-02-25
"Prince""musician"57true1958-06-072016-04-21
"Alan Rickman""actor"69false1946-02-212016-01-14
"Florence Henderson""actor"82true1934-02-142016-11-24
"Harper Lee""author"89false1926-04-282016-02-19
"Zsa Zsa Gábor""actor"99true1917-02-062016-12-18
"George Michael""musician"53false1963-06-252016-12-25
" + ], + "text/plain": [ + "shape: (10, 6)\n", + "┌────────────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────────┐\n", + "│ Lots of people ┆ __UNNAMED__1 ┆ __UNNAMED__2 ┆ __UNNAMED__3 ┆ __UNNAMED__4 ┆ __UNNAMED__5 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ i64 ┆ bool ┆ date ┆ date │\n", + "╞════════════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════════╡\n", + "│ David Bowie ┆ musician ┆ 69 ┆ true ┆ 1947-01-08 ┆ 2016-01-10 │\n", + "│ Carrie Fisher ┆ actor ┆ 60 ┆ true ┆ 1956-10-21 ┆ 2016-12-27 │\n", + "│ Chuck Berry ┆ musician ┆ 90 ┆ true ┆ 1926-10-18 ┆ 2017-03-18 │\n", + "│ Bill Paxton ┆ actor ┆ 61 ┆ true ┆ 1955-05-17 ┆ 2017-02-25 │\n", + "│ Prince ┆ musician ┆ 57 ┆ true ┆ 1958-06-07 ┆ 2016-04-21 │\n", + "│ Alan Rickman ┆ actor ┆ 69 ┆ false ┆ 1946-02-21 ┆ 2016-01-14 │\n", + "│ Florence Henderson ┆ actor ┆ 82 ┆ true ┆ 1934-02-14 ┆ 2016-11-24 │\n", + "│ Harper Lee ┆ author ┆ 89 ┆ false ┆ 1926-04-28 ┆ 2016-02-19 │\n", + "│ Zsa Zsa Gábor ┆ actor ┆ 99 ┆ true ┆ 1917-02-06 ┆ 2016-12-18 │\n", + "│ George Michael ┆ musician ┆ 53 ┆ false ┆ 1963-06-25 ┆ 2016-12-25 │\n", + "└────────────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┘" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pl.read_excel(\n", + " \"data/deaths.xlsx\",\n", + " read_options={\"skip_rows\": 4, \"n_rows\": 10},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "2d277c3a", + "metadata": {}, + "source": [ + "## Data Types\n", + "\n", + "In CSV files, all values are strings. This is not particularly true to the data, but it is simple: everything is a string.\n", + "\n", + "The underlying data in Excel spreadsheets is more complex. A cell can be one of five things:\n", + "\n", + "- A logical, like TRUE / FALSE\n", + "\n", + "- A number, like \"10\" or \"10.5\"\n", + "\n", + "- A date, which can also include time like \"11/1/21\" or \"11/1/21 3:00 PM\"\n", + "\n", + "- A string, like \"ten\"\n", + "\n", + "- A currency, which allows numeric values in a limited range and four decimal digits of fixed precision\n", + "\n", + "When working with spreadsheet data, it's important to keep in mind that how the underlying data is stored can be very different than what you see in the cell. For example, Excel has no notion of an integer. All numbers are stored as floating points (real number), but you can choose to display the data with a customizable number of decimal points. Similarly, dates are actually stored as numbers, specifically the number of seconds since January 1, 1970. You can customize how you display the date by applying formatting in Excel. Confusingly, it's also possible to have something that looks like a number but is actually a string (e.g. type `'10` into a cell in Excel).\n", + "\n", + "These differences between how the underlying data are stored vs. how they're displayed can cause surprises when the data are loaded into analytical tools such as **polars**. By default, **polars** will guess the data type in a given column.\n", + "A recommended workflow is to let **polars** guess the column types initially, inspect them, and then change any data types that you want to.\n" + ] + }, + { + "cell_type": "markdown", + "id": "0d73bbad", + "metadata": {}, + "source": [ + "## Writing to Excel\n", + "\n", + "Let's create a small data frame that we can then write out. Note that `item` is a category and `quantity` is an integer.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "15963e18", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (3, 2)
itemquantity
cati64
"brownie"10
"cupcake"5
"cookie"8
" + ], + "text/plain": [ + "shape: (3, 2)\n", + "┌─────────┬──────────┐\n", + "│ item ┆ quantity │\n", + "│ --- ┆ --- │\n", + "│ cat ┆ i64 │\n", + "╞═════════╪══════════╡\n", + "│ brownie ┆ 10 │\n", + "│ cupcake ┆ 5 │\n", + "│ cookie ┆ 8 │\n", + "└─────────┴──────────┘" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "bake_sale = pl.DataFrame(\n", + " {\n", + " \"item\": pl.Series([\"brownie\", \"cupcake\", \"cookie\"], dtype=pl.Categorical),\n", + " \"quantity\": [10, 5, 8],\n", + " }\n", + ")\n", + "bake_sale" + ] + }, + { + "cell_type": "markdown", + "id": "345bca3d", + "metadata": {}, + "source": [ + "You can write data back to disk as an Excel file using the `.write_excel()` method. Polars does not use a row index like pandas, so only the columns in the DataFrame are written by default.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "1fc17141", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "bake_sale.write_excel(\"data/bake_sale.xlsx\")" + ] + }, + { + "cell_type": "markdown", + "id": "f892c2e7", + "metadata": {}, + "source": [ + "The figure below shows what the data looks like in Excel.\n", + "\n", + "![Bake sale data frame created earlier in Excel.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-bake-sale.png)\n" + ] + }, + { + "cell_type": "markdown", + "id": "8d555c84", + "metadata": {}, + "source": [ + "Just like reading from a CSV, information on data type is lost when we read the data back in—you can see this if you read the data back in and check the `schema` for the data types. Although we kept `Int64` because **polars** recognised that the second column was of integer type, we lost the categorical data type for \"item\". This data type loss makes Excel files unreliable for caching interim results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "27e128f9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Schema([('item', String), ('quantity', Int64)])" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pl.read_excel(\"data/bake_sale.xlsx\").schema" + ] + }, + { + "cell_type": "markdown", + "id": "215634bf", + "metadata": {}, + "source": [ + "### Formatted Output\n", + "\n", + "If you need more formatting options and more control over how you write spreadsheets, check out the documentation for [openpyxl](https://openpyxl.readthedocs.io/) which can do pretty much everything you imagine. Generally, releasing data in spreadsheets is not the best option: but if you do want to release data in spreadsheets according to best practice, then check out [gptables](https://gptables.readthedocs.io/).\n" + ] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "-all", + "encoding": "# -*- coding: utf-8 -*-", + "formats": "md:myst", + "main_language": "python" + }, + "kernelspec": { + "display_name": "python4ds", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" }, - "nbformat": 4, - "nbformat_minor": 5 + "toc-showtags": true + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/webscraping-and-apis.ipynb b/webscraping-and-apis.ipynb index fc73aa9..4441554 100644 --- a/webscraping-and-apis.ipynb +++ b/webscraping-and-apis.ipynb @@ -81,11 +81,11 @@ "source": [ "import textwrap\n", "\n", + "import lets_plot as lp\n", "import pandas as pd\n", "import polars as pl\n", "import requests\n", - "from bs4 import BeautifulSoup\n", - "import lets_plot as lp" + "from bs4 import BeautifulSoup" ] }, { @@ -151,26 +151,29 @@ "# Convert dates using string operations\n", "df = (\n", " pl.DataFrame(json_data[\"months\"])\n", - " .with_columns([\n", - " # Add day to make it a valid date string\n", - " (pl.col(\"date\") + \"-01\").str.to_date(format=\"%Y %b-%d\").alias(\"date\"),\n", - " pl.col(\"value\").cast(pl.Float64).alias(\"value\"),\n", - " ])\n", + " .with_columns(\n", + " [\n", + " # Add day to make it a valid date string\n", + " (pl.col(\"date\") + \"-01\").str.to_date(format=\"%Y %b-%d\").alias(\"date\"),\n", + " pl.col(\"value\").cast(pl.Float64).alias(\"value\"),\n", + " ]\n", + " )\n", " .drop_nulls(\"date\")\n", " .sort(\"date\")\n", ")\n", "\n", "\n", - "\n", "# Initialize the library\n", "lp.LetsPlot.setup_html()\n", "\n", "# Create plot using the alias\n", - "chart = lp.ggplot(df, lp.aes(x=\"date\", y=\"value\")) + \\\n", - " lp.geom_line(size=2.0, color=\"steelblue\") + \\\n", - " lp.ggtitle(title) + \\\n", - " lp.ylim(0, df[\"value\"].max() * 1.2) + \\\n", - " lp.theme_classic()\n", + "chart = (\n", + " lp.ggplot(df, lp.aes(x=\"date\", y=\"value\"))\n", + " + lp.geom_line(size=2.0, color=\"steelblue\")\n", + " + lp.ggtitle(title)\n", + " + lp.ylim(0, df[\"value\"].max() * 1.2)\n", + " + lp.theme_classic()\n", + ")\n", "\n", "chart" ] @@ -223,9 +226,7 @@ " ).reset_index()\n", " )\n", " .rename({\"Country\": \"country\"})\n", - " .with_columns(\n", - " pl.col(\"country\").map_elements(lambda x: textwrap.fill(x, 10))\n", - " )\n", + " .with_columns(pl.col(\"country\").map_elements(lambda x: textwrap.fill(x, 10)))\n", " .sort(indicator_code, descending=True)\n", ")\n", "\n", @@ -523,8 +524,6 @@ "outputs": [], "source": [ "import polars as pl\n", - "import pandas as pd\n", - "\n", "\n", "pd_list = pd.read_html(\"https://webscraper.io/test-sites/tables\", match=\"First Name\")\n", "# Retrieve first entry from list of data frames\n", From 1dcd47b78126b338d3ccfd7d9b618d8a31e2c88c Mon Sep 17 00:00:00 2001 From: Nwabueze Ugoh Date: Fri, 17 Apr 2026 13:19:13 +0100 Subject: [PATCH 20/34] Update spreadsheet.ipynb and databases.ipynb to replace pandas with polars. --- databases.ipynb | 36 +- pyproject.toml | 2 + spreadsheets.ipynb | 907 +++++++++++++++++++++++---------------------- uv.lock | 26 ++ 4 files changed, 510 insertions(+), 461 deletions(-) diff --git a/databases.ipynb b/databases.ipynb index f14914c..b295748 100644 --- a/databases.ipynb +++ b/databases.ipynb @@ -17,7 +17,7 @@ "\n", "### Prerequisites\n", "\n", - "You will need the **pandas**, **SQLModel**, and **ibis** packages for this chapter. You probably already have **pandas** installed; to install **SQLModel** and **ibis** respectively run `uv add sqlmodel` and `uv add ibis-framework` on your computer's command line. First, let's bring in some general packages and turn off verbose warnings." + "You will need the **polars**, **SQLModel**, and **ibis** packages for this chapter. You probably already have **polars** installed; to install **SQLModel** and **ibis** respectively run `uv add sqlmodel` and `uv add ibis-framework` on your computer's command line. First, let's bring in some general packages and turn off verbose warnings." ] }, { @@ -39,10 +39,9 @@ "metadata": {}, "source": [ "## Database Basics\n", - "\n", - "At the simplest level, you can think about a database as a collection of data frames, called **tables** in database terminology.\n", - "Like a **pandas** data frame, a database table is a collection of named columns, where every value in the column is the same type.\n", - "There are three high level differences between data frames and database tables:\n", + "At the simplest level, you can think about a database as a collection of data frames, called **tables** in database terminology. \n", + "Like a **Polars** DataFrame, a database table is a collection of named columns, where every value in a column shares the same data type. \n", + "There are three high-level differences between data frames and database tables:\n", "\n", "- Database tables are stored on disk (ie on file) and can be arbitrarily large.\n", " Data frames are stored in memory, and are fundamentally limited (although that limit is still big enough for many problems). You can think about the difference between on disk and in memory as being like the difference between long-term and short-term memory (and you have much more limited capacity in the latter).\n", @@ -68,7 +67,7 @@ "\n", "- You'll always use a database interface that provides a connection to the database, for example Python's built-in **sqlite** package\n", "\n", - "- You'll also use a package that pushes and/or pulls data to/from the database, for example **pandas**\n", + "- You'll also use a package that pushes and/or pulls data to/from the database, for example **polars**\n", "\n", "The precise details of the connection varies a lot from DBMS to DBMS so unfortunately we can't cover all the details here. The initial setup will often take a little fiddling (and maybe some research) to get right, but you'll generally only need to do it once. We'll do the best we can to cover some basics here.\n", "\n", @@ -112,7 +111,7 @@ "id": "2992b718", "metadata": {}, "source": [ - "Note that the output here is in the form a Python object called a tuple. If we wanted to put this into a **pandas** data frame, we can just pass it straight in:" + "Note that the output here is in the form of a Python object called a tuple. If we want to convert this into a **Polars** DataFrame, we can pass it to `pl.DataFrame()`. When working with tuples, you may need to provide column names using the **schema** argument or specify **orient=\"row\"** so Polars correctly interprets the structure." ] }, { @@ -122,9 +121,11 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import polars as pl\n", + "\n", + "df = pl.DataFrame(rows, orient=\"row\")\n", "\n", - "pd.DataFrame(rows)" + "df" ] }, { @@ -316,9 +317,9 @@ "source": [ "### Joins\n", "\n", - "If you're familiar with joins in **pandas**, SQL joins are very similar. Let's see if we can join the 'album' and 'track' tables to find the *name* of the albums in the above query.\n", + "If you’re familiar with joins in **polars**, SQL joins are very similar. Let’s see if we can join the 'album' and 'track' tables to find the *name* of the albums in the above query.\n", "\n", - "Note that as soon as we have the *same* column names in more than one table, we need to specify the table we are referring to when we use that column name. There are different options for joins (eg `INNER`, `LEFT`) that you can find out more about [here](https://en.wikipedia.org/wiki/Join_(SQL)).\n" + "In polars, you use the `df.join()` method, which defaults to an \"inner\" join. Note that if you have the same column names in both tables, Polars will often append a suffix (like _right) to the duplicate names to keep them distinct, unless you specify otherwise. There are different options for joins (eg `INNER`, `LEFT`) that you can find out more about [here](https://en.wikipedia.org/wiki/Join_(SQL)).\n" ] }, { @@ -403,9 +404,9 @@ "id": "495f97e5", "metadata": {}, "source": [ - "## SQL with **pandas**\n", + "## SQL with **polars**\n", "\n", - "**pandas** is well-equipped for working with SQL. We can simply push the query we just created straight through using its `read_sql()` function—but bear in mind we need to pass in the connection we created to the database too:" + "**polars** is well-equipped for working with SQL. We can simply push the query we just created straight through using its `read_database()` function—but bear in mind we need to pass in the connection we created to the database too:" ] }, { @@ -415,7 +416,10 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_sql(sql_join, con)" + "df = pl.read_database(\n", + " query=sql_join, # your SQL query (string)\n", + " connection=con # your connection object (SQLAlchemy, psycopg2 cursor, etc.)\n", + ")" ] }, { @@ -435,7 +439,7 @@ "source": [ "## SQL with **ibis**\n", "\n", - "It's not exactly satisfactory to have to write out your SQL queries in text. What if we could create commands directly from **pandas** commands? You can't *quite* do that, but there's a package that gets you pretty close and it's called [**ibis**](https://ibis-project.org/). **ibis** is particularly useful when you are reading from a database and want to query it just like you would a **pandas** data frame.\n", + "It's not exactly satisfactory to have to write out your SQL queries in text. What if we could create commands directly from **polars** commands? You can't *quite* do that, but there's a package that gets you pretty close and it's called [**ibis**](https://ibis-project.org/). **ibis** is particularly useful when you are reading from a database and want to query it just like you would a **polars** data frame.\n", "\n", "**Ibis** can connect to local databases (eg a SQLite database), server-based databases (eg Postgres), or cloud-based databased (eg Google's BigQuery). The syntax to make a connection is, for example, `ibis.bigquery.connect`.\n", "\n", @@ -462,7 +466,7 @@ "id": "6dcd7d71", "metadata": {}, "source": [ - "Okay, now let's reproduce the following query: \"SELECT albumid, AVG(milliseconds)/1e3/60 FROM track GROUP BY albumid ORDER BY AVG(milliseconds) ASC LIMIT 5;\". We'll use a groupby, a mutate (which you can think of like **pandas**' assign statement), a sort, and then `limit()` to only show the first five entries." + "Okay, now let's reproduce the following query: \"SELECT albumid, AVG(milliseconds)/1e3/60 FROM track GROUP BY albumid ORDER BY AVG(milliseconds) ASC LIMIT 5;\". We'll use a group_by, a mutate (which you can think of like **polars** assign statement), a sort, and then `limit()` to only show the first five entries." ] }, { diff --git a/pyproject.toml b/pyproject.toml index 14ed0d8..7740186 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ readme = "README.md" requires-python = ">=3.12.0,<3.13" dependencies = [ "beautifulsoup4>=4.12.3", + "fastexcel>=0.19.0", "graphviz>=0.20.3", "ibis-framework[sqlite]>=9.5.0", "ipykernel>=6.29.5", @@ -36,6 +37,7 @@ dependencies = [ "toml>=0.10.2", "watermark>=2.5.0", "wbgapi>=1.0.14", + "xlsxwriter>=3.2.0", "yfinance>=1.2.1", ] diff --git a/spreadsheets.ipynb b/spreadsheets.ipynb index 12f1672..8053af7 100644 --- a/spreadsheets.ipynb +++ b/spreadsheets.ipynb @@ -1,447 +1,464 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "95f0a171", - "metadata": {}, - "source": [ - "# Spreadsheets {#sec-spreadsheets}\n", - "\n", - "## Introduction\n", - "\n", - "This chapter will show you how to work with spreadsheets, for example Microsoft Excel files, in Python. We already saw how to import csv (and tsv) files in @sec-data-import. In this chapter we will introduce you to tools for working with data in Excel spreadsheets and Google Sheets.\n", - "\n", - "If you or your collaborators are using spreadsheets for organising data that will be ingested by an analytical tool like Python, we recommend reading the paper \"Data Organization in Spreadsheets\" by Karl Broman and Kara Woo {cite}`broman2018data`. The best practices presented in this paper will save you much headache down the line when you import the data from a spreadsheet into Python to analyse and visualise. (For spreadsheets that are meant to be read by humans, we recommend the [good practice tables](https://github.com/best-practice-and-impact/gptables) package.)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "51a55374", - "metadata": { - "tags": [ - "remove-cell" - ] - }, - "outputs": [], - "source": [ - "# remove cell\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib_inline.backend_inline\n", - "\n", - "# Plot settings\n", - "plt.style.use(\"https://github.com/aeturrell/python4DS/raw/main/plot_style.txt\")\n", - "matplotlib_inline.backend_inline.set_matplotlib_formats(\"svg\")" - ] - }, - { - "cell_type": "markdown", - "id": "17575f3a", - "metadata": {}, - "source": [ - "### Prerequisites\n", - "\n", - "You will need to install the **pandas** package for this chapter. You will also need to install the **openpyxl** package by running `uv add openpyxl` in the terminal." - ] - }, - { - "cell_type": "markdown", - "id": "ff99055a", - "metadata": {}, - "source": [ - "## Reading Excel (and Similar) Files\n", - "\n", - "**pandas** can read in xls, xlsx, xlsm, xlsb, odf, ods, and odt files from your local filesystem or from a URL. It also supports an option to read a single sheet or a list of sheets.\n", - "\n", - "To show how this works, we'll work with an example spreadsheet called \"students.xlsx\". The figure below shows what the spreadsheet looks like.\n", - "\n", - "![A look at the students spreadsheet in Excel. The spreadsheet contains information on 6 students, their ID, full name, favourite food, meal plan, and age.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-students.png)" - ] - }, - { - "cell_type": "markdown", - "id": "29f2f4e0", - "metadata": {}, - "source": [ - "The first argument to `pd.read_excel()` is the path to the file to read. If you have downloaded the [file]() onto your computer and put it in a subfolder called \"data\" then you would want to use the path \"data/students.xlsx\" but we can also load it directly from the URL." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bf00d4c6", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "students = pd.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/students.xlsx\"\n", - ")\n", - "students" - ] - }, - { - "cell_type": "markdown", - "id": "03d6ba9c", - "metadata": {}, - "source": [ - "We have six students in the data and five variables on each student. However there are a few things we might want to address in this dataset:\n", - "\n", - "- The column names are all over the place. You can provide column names that follow a consistent format; we recommend `snake_case` using the `names` argument.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ecce26ea", - "metadata": {}, - "outputs": [], - "source": [ - "pd.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/students.xlsx\",\n", - " names=[\"student_id\", \"full_name\", \"favourite_food\", \"meal_plan\", \"age\"],\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "bb07ad4f", - "metadata": {}, - "source": [ - "\n", - "- `age` is read in as a column of objects, but it really should be numeric. Just like with `read_csv()`, you can supply a `dtype` argument to `read_excel()` and specify the data types for the columns of data you read in. Your options include `\"boolean\"`, `\"int\"`, `\"float\"`, `\"datetime\"`, `\"string\"`, and more. But we can see right away that this isn't going to work with the \"age\" column as it mixes numbers and text: so we first need to map its text to numbers." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "22e45cc2", - "metadata": {}, - "outputs": [], - "source": [ - "students = pd.read_excel(\n", - " \"data/students.xlsx\",\n", - " names=[\"student_id\", \"full_name\", \"favourite_food\", \"meal_plan\", \"age\"],\n", - ")\n", - "students[\"age\"] = students[\"age\"].replace(\"five\", 5)\n", - "students" - ] - }, - { - "cell_type": "markdown", - "id": "c8a07159", - "metadata": {}, - "source": [ - "Okay, now we can apply the data types." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f67490d3", - "metadata": {}, - "outputs": [], - "source": [ - "students = students.astype(\n", - " {\n", - " \"student_id\": \"Int64\",\n", - " \"full_name\": \"string\",\n", - " \"favourite_food\": \"string\",\n", - " \"meal_plan\": \"category\",\n", - " \"age\": \"Int64\",\n", - " }\n", - ")\n", - "students.info()" - ] - }, - { - "cell_type": "markdown", - "id": "362ff5a5", - "metadata": {}, - "source": [ - "It took multiple steps and trial-and-error to load the data in exactly the format we want, and this is not unexpected. Data science is an iterative process. There is no way to know exactly what the data will look like until you load it and take a look at it. The general pattern we used is load the data, take a peek, make adjustments to your code, load it again, and repeat until you're happy with the result." - ] - }, - { - "cell_type": "markdown", - "id": "8b00c2a4", - "metadata": {}, - "source": [ - "### Reading Individual Sheets\n", - "\n", - "An important feature that distinguishes spreadsheets from flat files is the notion of multiple sheets. The figure below shows an Excel spreadsheet with multiple sheets. The data come from the **palmerpenguins** dataset {cite}`horst2020palmerpenguins`. Each sheet contains information on penguins from a different island where data were collected.\n", - "\n", - "![A look at the penguins spreadsheet in Excel. The spreadsheet contains has three sheets: Torgersen Island, Biscoe Island, and Dream Island.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-penguins-islands.png)\n", - "\n", - "You can read a single sheet using the following command (so as not to show the whole file, we'll use `.head()` to just show the first 5 rows):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e37f9e3d", - "metadata": {}, - "outputs": [], - "source": [ - "pd.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/penguins.xlsx\",\n", - " sheet_name=\"Torgersen Island\",\n", - ").head()" - ] - }, - { - "cell_type": "markdown", - "id": "641f6831", - "metadata": {}, - "source": [ - "Now this relies on us knowing the names of the sheets in advance. There will be situations where you can to read in data without peeking into the Excel spreadsheet. To read all sheets in, use `sheet_name=None`. The object that's created is a dictionary with key value pairs that are sheet names and data frames respectively. Let's look at the second key value pair (note that we have to convert the keys() and values() objects to list to then retrieve the second element of each using a subscript, ie `list(dictionary.keys())[]`).\n", - "\n", - "To give a sense of how this works, let's first print all of the retrieved keys:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0f3a3b78", - "metadata": {}, - "outputs": [], - "source": [ - "penguins_dict = pd.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/penguins.xlsx\",\n", - " sheet_name=None,\n", - ")\n", - "print([x for x in penguins_dict.keys()])" - ] - }, - { - "cell_type": "markdown", - "id": "076f1ebe", - "metadata": {}, - "source": [ - "Now let's show the second entry data frame" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15495426", - "metadata": {}, - "outputs": [], - "source": [ - "print(list(penguins_dict.keys())[1])\n", - "list(penguins_dict.values())[1].head()" - ] - }, - { - "cell_type": "markdown", - "id": "536ab4bb", - "metadata": {}, - "source": [ - "What we really want is these three *consistent* datasets to be in the *same* single data frame. For this, we can use the `pd.concat()` function. This concatenates any given iterable of data frames." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "151ba846", - "metadata": {}, - "outputs": [], - "source": [ - "penguins = pd.concat(penguins_dict.values(), axis=0)\n", - "penguins" - ] - }, - { - "cell_type": "markdown", - "id": "aa458187", - "metadata": {}, - "source": [ - "### Reading Part of a Sheet\n", - "\n", - "Since many use Excel spreadsheets for presentation as well as for data storage, it's quite common to find cell entries in a spreadsheet that are not part of the data you want to read in.\n", - "\n", - "The figure below shows such a spreadsheet: in the middle of the sheet is what looks like a data frame but there is extraneous text in cells above and below the data.\n", - "\n", - "![A look at the deaths spreadsheet in Excel. The spreadsheet has four rows on top that contain non-data information; the text 'For the same of consistency in the data layout, which is really a beautiful thing, I will keep making notes up here.' is spread across cells in these top four rows. Then, there is a data frame that includes information on deaths of 10 famous people, including their names, professions, ages, whether they have kids or not, date of birth and death. At the bottom, there are four more rows of non-data information; the text 'This has been really fun, but we're signing off now!' is spread across cells in these bottom four rows.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-deaths.png)\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "id": "61505c86", - "metadata": {}, - "source": [ - "This spreadsheet can be downloaded from [here](https://github.com/aeturrell/python4DS/tree/main/data) or you can load it directly from a URL. If you want to load it from your own computer's disk, you'll need to save it in a sub-folder called \"data\" first.\n", - "\n", - "\n", - "The top three rows and the bottom four rows are not part of the data frame. We could skip the top three rows with `skiprows`. Note that we set `skiprows=4` since the fourth row contains column names, not the data.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "29987b9d", - "metadata": {}, - "outputs": [], - "source": [ - "pd.read_excel(\"data/deaths.xlsx\", skiprows=4)" - ] - }, - { - "cell_type": "markdown", - "id": "a1a8c3ca", - "metadata": {}, - "source": [ - "We could also set `nrows` to omit the extraneous rows at the bottom (another option would to be to skip a set number of rows at the end using `skipfooter`)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9d7a3db3", - "metadata": {}, - "outputs": [], - "source": [ - "pd.read_excel(\"data/deaths.xlsx\", skiprows=4, nrows=10)" - ] - }, - { - "cell_type": "markdown", - "id": "2d277c3a", - "metadata": {}, - "source": [ - "## Data Types\n", - "\n", - "In CSV files, all values are strings. This is not particularly true to the data, but it is simple: everything is a string.\n", - "\n", - "The underlying data in Excel spreadsheets is more complex. A cell can be one of five things:\n", - "\n", - "- A logical, like TRUE / FALSE\n", - "\n", - "- A number, like \"10\" or \"10.5\"\n", - "\n", - "- A date, which can also include time like \"11/1/21\" or \"11/1/21 3:00 PM\"\n", - "\n", - "- A string, like \"ten\"\n", - "\n", - "- A currency, which allows numeric values in a limited range and four decimal digits of fixed precision\n", - "\n", - "When working with spreadsheet data, it's important to keep in mind that how the underlying data is stored can be very different than what you see in the cell. For example, Excel has no notion of an integer. All numbers are stored as floating points (real number), but you can choose to display the data with a customizable number of decimal points. Similarly, dates are actually stored as numbers, specifically the number of seconds since January 1, 1970. You can customize how you display the date by applying formatting in Excel. Confusingly, it's also possible to have something that looks like a number but is actually a string (e.g. type `'10` into a cell in Excel).\n", - "\n", - "These differences between how the underlying data are stored vs. how they're displayed can cause surprises when the data are loaded into analytical tools such as **pandas**. By default, **pandas** will guess the data type in a given column.\n", - "A recommended workflow is to let **pandas** guess the column types initially, inspect them, and then change any data types that you want to." - ] - }, - { - "cell_type": "markdown", - "id": "0d73bbad", - "metadata": {}, - "source": [ - "## Writing to Excel\n", - "\n", - "Let's create a small data frame that we can then write out. Note that `item` is a category and `quantity` is an integer." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15963e18", - "metadata": {}, - "outputs": [], - "source": [ - "bake_sale = pd.DataFrame(\n", - " {\"item\": pd.Categorical([\"brownie\", \"cupcake\", \"cookie\"]), \"quantity\": [10, 5, 8]}\n", - ")\n", - "bake_sale" - ] - }, - { - "cell_type": "markdown", - "id": "345bca3d", - "metadata": {}, - "source": [ - "You can write data back to disk as an Excel file using the `.to_excel()` function. The `index=False` keyword argument just writes the two columns without the index that was automatically added in the last step." - ] - }, - { - "cell_type": "markdown", - "id": "1fc17141", - "metadata": {}, - "source": [ - "```python\n", - "bake_sale.to_excel(\"data/bake_sale.xlsx\", index=False)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "f892c2e7", - "metadata": {}, - "source": [ - "The figure below shows what the data looks like in Excel.\n", - "\n", - "![Bake sale data frame created earlier in Excel.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-bake-sale.png)" - ] - }, - { - "cell_type": "markdown", - "id": "8d555c84", - "metadata": {}, - "source": [ - "Just like reading from a CSV, information on data type is lost when we read the data back in—you can see this is you read the data back in and check the `info` for the data types. Although we kept `int64` because **pandas** recognise that the second column was of integer type, we lost the categorical data type for \"item\". This data type loss makes Excel files unreliable for caching interim results." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "27e128f9", - "metadata": {}, - "outputs": [], - "source": [ - "pd.read_excel(\"data/bake_sale.xlsx\").info()" - ] - }, - { - "cell_type": "markdown", - "id": "215634bf", - "metadata": {}, - "source": [ - "### Formatted Output\n", - "\n", - "If you need more formatting options and more control over how you write spreadsheets, check out the documentation for [openpyxl](https://openpyxl.readthedocs.io/) which can do pretty much everything you imagine. Generally, releasing data in spreadsheets is not the best option: but if you do want to release data in spreadsheets according to best practice, then check out [gptables](https://gptables.readthedocs.io/)." - ] - } - ], - "metadata": { - "interpreter": { - "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" - }, - "jupytext": { - "cell_metadata_filter": "-all", - "encoding": "# -*- coding: utf-8 -*-", - "formats": "md:myst", - "main_language": "python" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.12" - }, - "toc-showtags": true - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "id": "95f0a171", + "metadata": {}, + "source": [ + "# Spreadsheets {#sec-spreadsheets}\n", + "\n", + "## Introduction\n", + "\n", + "This chapter will show you how to work with spreadsheets, for example Microsoft Excel files, in Python. We already saw how to import csv (and tsv) files in @sec-data-import. In this chapter we will introduce you to tools for working with data in Excel spreadsheets and Google Sheets.\n", + "\n", + "If you or your collaborators are using spreadsheets for organising data that will be ingested by an analytical tool like Python, we recommend reading the paper \"Data Organization in Spreadsheets\" by Karl Broman and Kara Woo {cite}`broman2018data`. The best practices presented in this paper will save you much headache down the line when you import the data from a spreadsheet into Python to analyse and visualise. (For spreadsheets that are meant to be read by humans, we recommend the [good practice tables](https://github.com/best-practice-and-impact/gptables) package.)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "51a55374", + "metadata": { + "tags": [ + "remove-cell" + ] + }, + "outputs": [], + "source": [ + "# remove cell\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib_inline.backend_inline\n", + "\n", + "# Plot settings\n", + "plt.style.use(\"https://github.com/aeturrell/python4DS/raw/main/plot_style.txt\")\n", + "matplotlib_inline.backend_inline.set_matplotlib_formats(\"svg\")" + ] + }, + { + "cell_type": "markdown", + "id": "17575f3a", + "metadata": {}, + "source": [ + "### Prerequisites\n", + "\n", + "You will need the **polars** package for this chapter. Install **fastexcel** so `read_excel()` can use the default (fast) engine, **xlsxwriter** for `write_excel()`, and **openpyxl** if you want to use the **openpyxl** engine explicitly (`uv add fastexcel xlsxwriter openpyxl`)." + ] + }, + { + "cell_type": "markdown", + "id": "ff99055a", + "metadata": {}, + "source": [ + "## Reading Excel (and Similar) Files\n", + "\n", + "**polars** can read in xls, xlsx, xlsm, xlsb, odf, ods, and odt files from your local filesystem or from a URL. It also supports an option to read a single sheet or a list of sheets. The default reader uses **fastexcel** (Rust-backed, via the **fastexcel** Python package); you can also select other engines such as **openpyxl** when you need engine-specific options.\n", + "\n", + "To show how this works, we'll work with an example spreadsheet called \"students.xlsx\". The figure below shows what the spreadsheet looks like.\n", + "\n", + "![A look at the students spreadsheet in Excel. The spreadsheet contains information on 6 students, their ID, full name, favourite food, meal plan, and age.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-students.png)" + ] + }, + { + "cell_type": "markdown", + "id": "29f2f4e0", + "metadata": {}, + "source": [ + "The first argument to `pl.read_excel()` is the path to the file to read. If you have downloaded the [file]() onto your computer and put it in a subfolder called \"data\" then you would want to use the path \"data/students.xlsx\" but we can also load it directly from the URL." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf00d4c6", + "metadata": {}, + "outputs": [], + "source": [ + "import polars as pl\n", + "\n", + "students = pl.read_excel(\n", + " \"https://github.com/aeturrell/python4DS/raw/main/data/students.xlsx\",\n", + ")\n", + "students" + ] + }, + { + "cell_type": "markdown", + "id": "03d6ba9c", + "metadata": {}, + "source": [ + "We have six students in the data and five variables on each student. However there are a few things we might want to address in this dataset:\n", + "\n", + "- The column names are all over the place. You can rename them to follow a consistent format; we recommend `snake_case`. If you want to replace **every** column name in order, assigning to `students.columns` is clear and short. If you only rename some columns, use `.rename({\"Old Name\": \"new_name\", ...})` with the exact strings from the sheet.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ecce26ea", + "metadata": {}, + "outputs": [], + "source": [ + "students.columns = [\n", + " \"student_id\",\n", + " \"full_name\",\n", + " \"favourite_food\",\n", + " \"meal_plan\",\n", + " \"age\",\n", + "]\n", + "students" + ] + }, + { + "cell_type": "markdown", + "id": "bb07ad4f", + "metadata": {}, + "source": [ + "\n", + "- `age` may be inferred as strings (for example **Utf8**) when the column mixes numeric values and text, but we want it numeric. Just like with `read_csv()`, you can supply a `schema_overrides` argument to `read_excel()` and specify Polars data types for the columns you read in (for example `pl.Int64`, `pl.Utf8`, `pl.Boolean`, `pl.Datetime`, and more). That still will not fix a value like `\"five\"` until we map it to a number first.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22e45cc2", + "metadata": {}, + "outputs": [], + "source": [ + "students = pl.read_excel(\"data/students.xlsx\")\n", + "students.columns = [\n", + " \"student_id\",\n", + " \"full_name\",\n", + " \"favourite_food\",\n", + " \"meal_plan\",\n", + " \"age\",\n", + "]\n", + "students = students.with_columns(pl.col(\"age\").replace({\"five\": 5}))\n", + "students" + ] + }, + { + "cell_type": "markdown", + "id": "c8a07159", + "metadata": {}, + "source": [ + "Okay, now we can apply the data types." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f67490d3", + "metadata": {}, + "outputs": [], + "source": [ + "students = students.with_columns(\n", + " [\n", + " pl.col(\"student_id\").cast(pl.Int64),\n", + " pl.col(\"full_name\").cast(pl.Utf8),\n", + " pl.col(\"favourite_food\").cast(pl.Utf8),\n", + " pl.col(\"meal_plan\").cast(pl.Categorical),\n", + " pl.col(\"age\").cast(pl.Int64),\n", + " ]\n", + ")\n", + "students.schema" + ] + }, + { + "cell_type": "markdown", + "id": "362ff5a5", + "metadata": {}, + "source": [ + "It took multiple steps and trial-and-error to load the data in exactly the format we want, and this is not unexpected. Data science is an iterative process. There is no way to know exactly what the data will look like until you load it and take a look at it. The general pattern we used is load the data, take a peek, make adjustments to your code, load it again, and repeat until you're happy with the result." + ] + }, + { + "cell_type": "markdown", + "id": "8b00c2a4", + "metadata": {}, + "source": [ + "### Reading Individual Sheets\n", + "\n", + "An important feature that distinguishes spreadsheets from flat files is the notion of multiple sheets. The figure below shows an Excel spreadsheet with multiple sheets. The data come from the **palmerpenguins** dataset {cite}`horst2020palmerpenguins`. Each sheet contains information on penguins from a different island where data were collected.\n", + "\n", + "![A look at the penguins spreadsheet in Excel. The spreadsheet contains has three sheets: Torgersen Island, Biscoe Island, and Dream Island.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-penguins-islands.png)\n", + "\n", + "You can read a single sheet using the following command (so as not to show the whole file, we'll use `.head()` to just show the first 5 rows):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e37f9e3d", + "metadata": {}, + "outputs": [], + "source": [ + "pl.read_excel(\n", + " \"https://github.com/aeturrell/python4DS/raw/main/data/penguins.xlsx\",\n", + " sheet_name=\"Torgersen Island\",\n", + ").head()" + ] + }, + { + "cell_type": "markdown", + "id": "641f6831", + "metadata": {}, + "source": [ + "Now this relies on us knowing the names of the sheets in advance. There will be situations where you want to read in data without peeking into the Excel spreadsheet. To read all sheets in Polars, use `sheet_id=0` (or `sheet_name=None`, which also works in recent versions of Polars). The object that’s created is a dictionary where the keys are the sheet names and the values are Polars DataFrames. To access a specific sheet, you can convert the keys() or values() to a list and then index into it, ie `list(dictionary.keys())[]` .\n", + "\n", + "To give a sense of how this works, let's first print all of the retrieved keys:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f3a3b78", + "metadata": {}, + "outputs": [], + "source": [ + "penguins_dict = pl.read_excel(\n", + " \"https://github.com/aeturrell/python4DS/raw/main/data/penguins.xlsx\",\n", + " sheet_id=0,\n", + ")\n", + "print([x for x in penguins_dict.keys()])" + ] + }, + { + "cell_type": "markdown", + "id": "076f1ebe", + "metadata": {}, + "source": [ + "Now let's show the second entry data frame" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15495426", + "metadata": {}, + "outputs": [], + "source": [ + "print(list(penguins_dict.keys())[1])\n", + "list(penguins_dict.values())[1].head()" + ] + }, + { + "cell_type": "markdown", + "id": "536ab4bb", + "metadata": {}, + "source": [ + "What we really want is these three *consistent* datasets to be in the *same* single data frame. For this, we can use the `pl.concat()` function. This concatenates any given iterable of data frames." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "151ba846", + "metadata": {}, + "outputs": [], + "source": [ + "penguins = pl.concat(penguins_dict.values())\n", + "penguins" + ] + }, + { + "cell_type": "markdown", + "id": "aa458187", + "metadata": {}, + "source": [ + "### Reading Part of a Sheet\n", + "\n", + "Since many use Excel spreadsheets for presentation as well as for data storage, it's quite common to find cell entries in a spreadsheet that are not part of the data you want to read in.\n", + "\n", + "The figure below shows such a spreadsheet: in the middle of the sheet is what looks like a data frame but there is extraneous text in cells above and below the data.\n", + "\n", + "![A look at the deaths spreadsheet in Excel. The spreadsheet has four rows on top that contain non-data information; the text 'For the same of consistency in the data layout, which is really a beautiful thing, I will keep making notes up here.' is spread across cells in these top four rows. Then, there is a data frame that includes information on deaths of 10 famous people, including their names, professions, ages, whether they have kids or not, date of birth and death. At the bottom, there are four more rows of non-data information; the text 'This has been really fun, but we're signing off now!' is spread across cells in these bottom four rows.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-deaths.png)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "61505c86", + "metadata": {}, + "source": [ + "This spreadsheet can be downloaded from [here](https://github.com/aeturrell/python4DS/tree/main/data) or you can load it directly from a URL. If you want to load it from your own computer's disk, you'll need to save it in a sub-folder called \"data\" first.\n", + "\n", + "\n", + "The top three rows and the bottom four rows are not part of the data frame. We could skip the top three rows by passing `read_options` to `read_excel()`. Note that we set `skip_rows=4` since the fourth row contains column names, not the data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29987b9d", + "metadata": {}, + "outputs": [], + "source": [ + "pl.read_excel(\n", + " \"data/deaths.xlsx\",\n", + " read_options={\"skip_rows\": 4},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "a1a8c3ca", + "metadata": {}, + "source": [ + "We could also set `n_rows` inside `read_options` to omit the extraneous rows at the bottom (another option would be to skip a set number of rows at the end using `skip_footer` in `read_options`, depending on the engine)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d7a3db3", + "metadata": {}, + "outputs": [], + "source": [ + "pl.read_excel(\n", + " \"data/deaths.xlsx\",\n", + " read_options={\"skip_rows\": 4, \"n_rows\": 10},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "2d277c3a", + "metadata": {}, + "source": [ + "## Data Types\n", + "\n", + "In CSV files, all values are strings. This is not particularly true to the data, but it is simple: everything is a string.\n", + "\n", + "The underlying data in Excel spreadsheets is more complex. A cell can be one of five things:\n", + "\n", + "- A logical, like TRUE / FALSE\n", + "\n", + "- A number, like \"10\" or \"10.5\"\n", + "\n", + "- A date, which can also include time like \"11/1/21\" or \"11/1/21 3:00 PM\"\n", + "\n", + "- A string, like \"ten\"\n", + "\n", + "- A currency, which allows numeric values in a limited range and four decimal digits of fixed precision\n", + "\n", + "When working with spreadsheet data, it's important to keep in mind that how the underlying data is stored can be very different than what you see in the cell. For example, Excel has no notion of an integer. All numbers are stored as floating points (real number), but you can choose to display the data with a customizable number of decimal points. Similarly, dates are actually stored as numbers, specifically the number of seconds since January 1, 1970. You can customize how you display the date by applying formatting in Excel. Confusingly, it's also possible to have something that looks like a number but is actually a string (e.g. type `'10` into a cell in Excel).\n", + "\n", + "These differences between how the underlying data are stored vs. how they're displayed can cause surprises when the data are loaded into analytical tools such as **polars**. By default, **polars** will guess the data type in a given column.\n", + "A recommended workflow is to let **polars** guess the column types initially, inspect them, and then change any data types that you want to." + ] + }, + { + "cell_type": "markdown", + "id": "0d73bbad", + "metadata": {}, + "source": [ + "## Writing to Excel\n", + "\n", + "Let's create a small data frame that we can then write out. Note that `item` is a category and `quantity` is an integer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15963e18", + "metadata": {}, + "outputs": [], + "source": [ + "bake_sale = pl.DataFrame(\n", + " {\n", + " \"item\": pl.Series([\"brownie\", \"cupcake\", \"cookie\"], dtype=pl.Categorical),\n", + " \"quantity\": [10, 5, 8],\n", + " }\n", + ")\n", + "bake_sale" + ] + }, + { + "cell_type": "markdown", + "id": "345bca3d", + "metadata": {}, + "source": [ + "You can write data back to disk as an Excel file using the `.write_excel()` method. Polars does not use a row index like pandas, so only the columns in the DataFrame are written by default." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1fc17141", + "metadata": {}, + "outputs": [], + "source": [ + "bake_sale.write_excel(\"data/bake_sale.xlsx\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "f892c2e7", + "metadata": {}, + "source": [ + "The figure below shows what the data looks like in Excel.\n", + "\n", + "![Bake sale data frame created earlier in Excel.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-bake-sale.png)" + ] + }, + { + "cell_type": "markdown", + "id": "8d555c84", + "metadata": {}, + "source": [ + "Just like reading from a CSV, information on data type is lost when we read the data back in—you can see this if you read the data back in and check the `schema` for the data types. Although we kept `Int64` because **polars** recognised that the second column was of integer type, we lost the categorical data type for \"item\". This data type loss makes Excel files unreliable for caching interim results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27e128f9", + "metadata": {}, + "outputs": [], + "source": [ + "pl.read_excel(\"data/bake_sale.xlsx\").schema" + ] + }, + { + "cell_type": "markdown", + "id": "215634bf", + "metadata": {}, + "source": [ + "### Formatted Output\n", + "\n", + "If you need more formatting options and more control over how you write spreadsheets, check out the documentation for [openpyxl](https://openpyxl.readthedocs.io/) which can do pretty much everything you imagine. Generally, releasing data in spreadsheets is not the best option: but if you do want to release data in spreadsheets according to best practice, then check out [gptables](https://gptables.readthedocs.io/)." + ] + } + ], + "metadata": { + "interpreter": { + "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" + }, + "jupytext": { + "cell_metadata_filter": "-all", + "encoding": "# -*- coding: utf-8 -*-", + "formats": "md:myst", + "main_language": "python" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + }, + "toc-showtags": true + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/uv.lock b/uv.lock index 5342f08..8e7d9ad 100644 --- a/uv.lock +++ b/uv.lock @@ -354,6 +354,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/fd/afcd0496feca3276f509df3dbd5dae726fcc756f1a08d9e25abe1733f962/executing-2.1.0-py2.py3-none-any.whl", hash = "sha256:8d63781349375b5ebccc3142f4b30350c0cd9c79f921cde38be2be4637e98eaf", size = 25805, upload-time = "2024-09-01T12:37:33.007Z" }, ] +[[package]] +name = "fastexcel" +version = "0.19.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/c8/3b09911348e9c64dbf41096d3e8f0e93c141a23990ec9f32514111bd5f55/fastexcel-0.19.0.tar.gz", hash = "sha256:216c3719ee90963bd93a0bf8c10b177233046ac975b67651152fdaedd3c99aa1", size = 60323, upload-time = "2026-01-20T11:17:37.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/e0/3820e93ea606549cfddb8c437141dd69f2b245e74785efc8bd7511ba909d/fastexcel-0.19.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:68601072a0b4b4277c165b68f1055f88ef7ffe7ed6f08c1eeda0f0271e3f7da0", size = 3082362, upload-time = "2026-01-20T11:17:27.157Z" }, + { url = "https://files.pythonhosted.org/packages/66/0f/b42dc09515879192919942157292912393584045fd8bad98bd92961d4c30/fastexcel-0.19.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c8a87d94445678e7e3f46a6aa39d2afaee5b88a983ec3661143a6488d8955f44", size = 2864365, upload-time = "2026-01-20T11:17:28.786Z" }, + { url = "https://files.pythonhosted.org/packages/8e/4a/bc358b20fcff64b4c14ff7d7a0e1f797792b8b77e30ae755873c02362538/fastexcel-0.19.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e94fc1be6642555f277af792c22a9f80ec9b4d640d9690f00abb822b6d865069", size = 3186426, upload-time = "2026-01-20T11:17:19.087Z" }, + { url = "https://files.pythonhosted.org/packages/58/ae/d2ffdc5ad14190153e2422fc90a1052a4b0c3086d24cb8ae8967575321d8/fastexcel-0.19.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:334f9f40cd68b5924a712b6c104949757a0b8ad8a7e3fa3f3fad1c1ebc00258b", size = 3365628, upload-time = "2026-01-20T11:17:21.116Z" }, + { url = "https://files.pythonhosted.org/packages/6e/67/5f6d4e7760dc3dd8244cd124dabdd5bb7622bf1197edcc2513648847690e/fastexcel-0.19.0-cp310-abi3-win_amd64.whl", hash = "sha256:fbbdf9de79c3ef3572809bb187927c0dc5840968ffe513ea015a383024b7c6b0", size = 2905173, upload-time = "2026-01-20T11:17:33.687Z" }, +] + [[package]] name = "fastjsonschema" version = "2.21.1" @@ -1729,6 +1742,7 @@ version = "0.0.1" source = { virtual = "." } dependencies = [ { name = "beautifulsoup4" }, + { name = "fastexcel" }, { name = "graphviz" }, { name = "ibis-framework", extra = ["sqlite"] }, { name = "ipykernel" }, @@ -1759,12 +1773,14 @@ dependencies = [ { name = "toml" }, { name = "watermark" }, { name = "wbgapi" }, + { name = "xlsxwriter" }, { name = "yfinance" }, ] [package.metadata] requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.12.3" }, + { name = "fastexcel", specifier = ">=0.19.0" }, { name = "graphviz", specifier = ">=0.20.3" }, { name = "ibis-framework", extras = ["sqlite"], specifier = ">=9.5.0" }, { name = "ipykernel", specifier = ">=6.29.5" }, @@ -1795,6 +1811,7 @@ requires-dist = [ { name = "toml", specifier = ">=0.10.2" }, { name = "watermark", specifier = ">=2.5.0" }, { name = "wbgapi", specifier = ">=1.0.14" }, + { name = "xlsxwriter", specifier = ">=3.2.0" }, { name = "yfinance", specifier = ">=1.2.1" }, ] @@ -2463,6 +2480,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/21/02/88b65cc394961a60c43c70517066b6b679738caf78506a5da7b88ffcb643/widgetsnbextension-4.0.13-py3-none-any.whl", hash = "sha256:74b2692e8500525cc38c2b877236ba51d34541e6385eeed5aec15a70f88a6c71", size = 2335872, upload-time = "2024-08-22T12:18:19.491Z" }, ] +[[package]] +name = "xlsxwriter" +version = "3.2.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/2c/c06ef49dc36e7954e55b802a8b231770d286a9758b3d936bd1e04ce5ba88/xlsxwriter-3.2.9.tar.gz", hash = "sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c", size = 215940, upload-time = "2025-09-16T00:16:21.63Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/0c/3662f4a66880196a590b202f0db82d919dd2f89e99a27fadef91c4a33d41/xlsxwriter-3.2.9-py3-none-any.whl", hash = "sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3", size = 175315, upload-time = "2025-09-16T00:16:20.108Z" }, +] + [[package]] name = "yfinance" version = "1.2.1" From 369bd4597cdc9d34e3b9790494845337c3af665d Mon Sep 17 00:00:00 2001 From: Nwabueze Ugoh Date: Wed, 22 Apr 2026 15:11:15 +0100 Subject: [PATCH 21/34] Replace pandas with polars in rectangling.ipynb and webscraping-and-apis.ipynb, updating relevant code and documentation accordingly. --- rectangling.ipynb | 34 ++-- visualise.quarto_ipynb_1 | 136 ++++++++++++++++ webscraping-and-apis.ipynb | 130 +++++++-------- workflow-help.quarto_ipynb_1 | 115 ++++++++++++++ ...w-packages-and-environments.quarto_ipynb_1 | 149 ++++++++++++++++++ 5 files changed, 483 insertions(+), 81 deletions(-) create mode 100644 visualise.quarto_ipynb_1 create mode 100644 workflow-help.quarto_ipynb_1 create mode 100644 workflow-packages-and-environments.quarto_ipynb_1 diff --git a/rectangling.ipynb b/rectangling.ipynb index f0bcf47..09d3402 100644 --- a/rectangling.ipynb +++ b/rectangling.ipynb @@ -41,7 +41,7 @@ "source": [ "### Prerequisites\n", "\n", - "This chapter will use the **pandas** data analysis package." + "This chapter will use the **polars** data analysis package." ] }, { @@ -292,9 +292,9 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import polars as pl\n", "\n", - "pd.DataFrame(data=list_of_lists, columns=[\"a\", \"b\", \"c\"])" + "df = pl.DataFrame(data=list_of_lists, schema=[\"a\", \"b\", \"c\"])" ] }, { @@ -312,12 +312,13 @@ "metadata": {}, "outputs": [], "source": [ - "df = pd.DataFrame(\n", + "df = pl.DataFrame(\n", " {\n", - " \"alpha\": [[0, 1, 2], \"foo\", [], [3, 4]],\n", - " \"beta\": 1,\n", - " \"gamma\": [[\"a\", \"b\", \"c\"], pd.NA, [], [\"d\", \"e\"]],\n", - " }\n", + " \"alpha\": [[\"0,1,2\"], \"foo\", [], [\"3,4\"]],\n", + " \"beta\": [1, 1, 1, 1],\n", + " \"gamma\": [[\"a\", \"b\", \"c\"], None, [], [\"d\", \"e\"]],\n", + " },\n", + " strict=False,\n", ")\n", "df" ] @@ -540,7 +541,7 @@ "source": [ "### From JSON data to Data Frame\n", "\n", - "**pandas** has lots of options for turning JSON or dictionary data into a data frame. You do need to think a little bit about the structure of the data underneath though:\n" + "**polars** has lots of options for turning JSON or dictionary data into a data frame. You do need to think a little bit about the structure of the data underneath though:\n" ] }, { @@ -550,9 +551,9 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import polars as pl\n", "\n", - "pd.DataFrame(result[\"toppings\"], columns=[\"id\", \"type\"])" + "df = pl.DataFrame(result[\"toppings\"], schema=[\"id\", \"type\"])" ] }, { @@ -560,7 +561,7 @@ "id": "a1346020", "metadata": {}, "source": [ - "The web-scraped data we downloaded earlier had a more complicated structure, but **pandas** has a `json_normalize()` function that can cope with this. For example, with the following data, there are many missing entries but `json_normalize()` can still parse it into a Data Frame." + "The web-scraped data we downloaded earlier had a more complicated structure, but **polars** has a `json_normalize()` function that can cope with this. For example, with the following data, there are many missing entries but `json_normalize()` can still parse it into a Data Frame." ] }, { @@ -575,7 +576,8 @@ " {\"name\": {\"given\": \"Mark\", \"family\": \"Regner\"}},\n", " {\"id\": 2, \"name\": \"Faye Raker\"},\n", "]\n", - "pd.json_normalize(data)" + "pl.json_normalize(data)\n", + "\n" ] }, { @@ -583,7 +585,7 @@ "id": "7eaf00e1", "metadata": {}, "source": [ - "And we can control the level that properties like 'name' are split out to as well (you can check out more options over at the [**pandas** documentation](https://pandas.pydata.org/docs/reference/api/pandas.json_normalize.html))" + "And we can control the level that properties like 'name' are split out to as well (you can check out more options over at the [**polars** documentation](https://docs.pola.rs/api/python/stable/reference/api/polars.json_normalize.html))" ] }, { @@ -593,7 +595,7 @@ "metadata": {}, "outputs": [], "source": [ - "pd.json_normalize(data, max_level=0)" + "pl.json_normalize(data, max_level=0)" ] }, { @@ -601,7 +603,7 @@ "id": "78d637e5", "metadata": {}, "source": [ - "As well as the JSON normalise function, **pandas** has a `from_dict()` method to work with simpler dictionary objects." + "As well as the JSON normalise function, **polars** has a `from_dict()` method to work with simpler dictionary objects." ] } ], diff --git a/visualise.quarto_ipynb_1 b/visualise.quarto_ipynb_1 new file mode 100644 index 0000000..eb958d5 --- /dev/null +++ b/visualise.quarto_ipynb_1 @@ -0,0 +1,136 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Visualisation {#sec-visualise}\n", + "\n", + "After reading the first part of the book, you understand the basics of the most important tools for doing data science. Now it’s time to start diving into the details. In this part of the book, you’ll learn about visualising data in further depth (in @sec-vis-layers), and get further stuck into the details of the different kinds of data visualisation (in @sec-exploratory-data-analysis and @sec-communicate-plots). In this short chapter, we discuss the different ways to create visualisations, and the different purposes of visualisations.\n", + "\n", + "## Philosophies of data visualisation\n", + "\n", + "There are broadly two categories of approach to using code to create data visualisations: *imperative* (build what you want from individual elements) and *declarative* (say what you want from a list of pre-existing options). Choosing which to use involves a trade-off: imperative libraries offer you flexibility but at the cost of some verbosity; declarative libraries offer you a quick way to plot your data, but only if it’s in the right format to begin with, and customisation to special chart types is more difficult.\n", + "\n", + "Python has many excellent plotting packages, including perhaps the most powerful imperative plotting package around, **matplotlib**, and an amazing declarative library that we already saw, **lets-plot**. These two libraries will get you a long way, and each could be worthy of an entire book themselves. Fortunately for us, though, we can do 95% of what we need with a small number of commands from one or the other of them. In general, to keep this book as light as possible, we've opted to use **lets-plot** wherever possible—and @sec-vis-layers is going to take you on a more in-depth tour of how to use it yourself.\n", + "\n", + "## Purposes of data visualisation\n", + "\n", + "Data visualisation has all kinds of different purposes. It can be useful to bear in mind three broad categories of visualisation that are out there:\n", + "\n", + "- exploratory\n", + "- scientific\n", + "- narrative\n", + "\n", + "Let's look at each in a bit more detail.\n", + "\n", + "### Exploratory Data Viz\n", + "\n", + "The first of the three kinds is *exploratory data visualisation*, and it's the kind that you do when you're looking and data and trying to understand it. Just plotting the data is a really good strategy for getting a feel for any issues there might be. This is perhaps most famously demonstrated by Anscombe's quartet: four different datasets with the same mean, standard deviation, and correlation but very different data distributions." + ], + "id": "2d4a4a09" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "#| echo: false\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib_inline.backend_inline\n", + "\n", + "# Plot settings\n", + "plt.style.use(\"https://github.com/aeturrell/python4DS/raw/main/plot_style.txt\")\n", + "matplotlib_inline.backend_inline.set_matplotlib_formats(\"svg\")\n", + "\n", + "# Set max rows displayed for readability\n", + "pd.set_option(\"display.max_rows\", 6)\n", + "\n", + "x = [10, 8, 13, 9, 11, 14, 6, 4, 12, 7, 5]\n", + "y1 = [8.04, 6.95, 7.58, 8.81, 8.33, 9.96, 7.24, 4.26, 10.84, 4.82, 5.68]\n", + "y2 = [9.14, 8.14, 8.74, 8.77, 9.26, 8.10, 6.13, 3.10, 9.13, 7.26, 4.74]\n", + "y3 = [7.46, 6.77, 12.74, 7.11, 7.81, 8.84, 6.08, 5.39, 8.15, 6.42, 5.73]\n", + "x4 = [8, 8, 8, 8, 8, 8, 8, 19, 8, 8, 8]\n", + "y4 = [6.58, 5.76, 7.71, 8.84, 8.47, 7.04, 5.25, 12.50, 5.56, 7.91, 6.89]\n", + "\n", + "datasets = {\"I\": (x, y1), \"II\": (x, y2), \"III\": (x, y3), \"IV\": (x4, y4)}\n", + "\n", + "fig, axs = plt.subplots(\n", + " 2,\n", + " 2,\n", + " sharex=True,\n", + " sharey=True,\n", + " figsize=(10, 6),\n", + " gridspec_kw={\"wspace\": 0.08, \"hspace\": 0.08},\n", + ")\n", + "axs[0, 0].set(xlim=(0, 20), ylim=(2, 14))\n", + "axs[0, 0].set(xticks=(0, 10, 20), yticks=(4, 8, 12))\n", + "\n", + "for ax, (label, (x, y)) in zip(axs.flat, datasets.items()):\n", + " ax.text(0.1, 0.9, label, fontsize=20, transform=ax.transAxes, va=\"top\")\n", + " ax.tick_params(direction=\"in\", top=True, right=True)\n", + " ax.plot(x, y, \"o\")\n", + "\n", + " # linear regression\n", + " p1, p0 = np.polyfit(x, y, deg=1) # slope, intercept\n", + " ax.axline(xy1=(0, p0), slope=p1, color=\"r\", lw=2)\n", + "\n", + " # add text box for the statistics\n", + " stats = (\n", + " f\"$\\\\mu$ = {np.mean(y):.2f}\\n\"\n", + " f\"$\\\\sigma$ = {np.std(y):.2f}\\n\"\n", + " f\"$r$ = {np.corrcoef(x, y)[0][1]:.2f}\"\n", + " )\n", + " bbox = dict(boxstyle=\"round\", fc=\"blanchedalmond\", ec=\"orange\", alpha=0.5)\n", + " ax.text(\n", + " 0.95,\n", + " 0.07,\n", + " stats,\n", + " fontsize=9,\n", + " bbox=bbox,\n", + " transform=ax.transAxes,\n", + " horizontalalignment=\"right\",\n", + " )\n", + "\n", + "plt.suptitle(\"Anscombe's Quartet\")\n", + "plt.show()" + ], + "id": "c38c38fa", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Exploratory visualisation is usually quick and dirty, and flexible too. Some exploratory data viz can be automated, and there's a whole host of packages to help with this, including [**skimpy**](https://aeturrell.github.io/skimpy/).\n", + "\n", + "Beyond you and perhaps your co-authors/collaborators, however, not many other people should be seeing your exploratory visualisation! They will typically be worked up quickly, be numerous, and be throw-away. We'll look more at this in @sec-exploratory-data-analysis.\n", + "\n", + "### Scientific Data Viz\n", + "\n", + "The second kind, scientific data visualisation, is the prime cut of your exploratory visualisation. It's the kind of plot you might include in a more technical paper, the picture that says a thousand words. I often think of the first image of a black hole @akiyama2019first as a prime example of this. You can get away with having a high density of information in a scientific plot and, in short format journals, you may need to. The journal Physical Review Letters, which has an 8 page limit, has a classic of this genre in more or less every issue. Ensuring that important values can be accurately read from the plot is especially important in these kinds of charts. But they can also be the kind of plot that presents the killer results in a study; they might not be exciting to people who don't look at charts for a living, but they might be exciting and, just as importantly, understandable by your peers.\n", + "\n", + "This type of visualisation is especially popular in the big science journals like *Nature* and *Science*, where space is at a premium. We won't cover this type of plot in this book, because it tends to be very bespoke.\n", + "\n", + "### Narrative Data Viz\n", + "\n", + "The third and final kind is narrative data visualisation. This is the one that requires the most thought in the step where you go from the first view to the end product. It's a visualisation that doesn't just show a picture, but gives an insight. These are the kind of visualisations that you might see in the *Financial Times*, *The Economist*, or on the *BBC News* website. They come with aids that help the viewer focus on the aspects that the creator wanted them to (you can think of these aids or focuses as doing for visualisation what bold font does for text). They're well worth using in your work, especially if you're trying to communicate a particular narrative, and especially if the people you're communicating with don't have deep knowledge of the topic. You might use them in a paper that you hope will have a wide readership, in a blog post summarising your work, or in a report intended for a policymaker.\n", + "\n", + "You can find more information on the topic of communicating via data visualisations in the @sec-communicate-plots chapter." + ], + "id": "88d12184" + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "language": "python", + "display_name": "Python 3 (ipykernel)", + "path": "/Users/omagic/Documents/GitHub/python4DSpolars/.venv/share/jupyter/kernels/python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/webscraping-and-apis.ipynb b/webscraping-and-apis.ipynb index 9366eb6..6e8daf7 100644 --- a/webscraping-and-apis.ipynb +++ b/webscraping-and-apis.ipynb @@ -67,7 +67,7 @@ "source": [ "### Prerequisites\n", "\n", - "You will need to install the **pandas** package for this chapter. We'll use **seaborn** too, which you should already have installed. You will also need to install the **beautifulsoup**, **pandas-datareader**, and **wbgapi** packages in your terminal using `uv add beautifulsoup4`, `uv add pandas-datareader`, and `uv add wbgapi` respectively. We'll also use two built-in packages, **textwrap** and **requests**.\n", + "You will need to install the **pandas** and **polars** package for this chapter. We'll use **seaborn** too, which you should already have installed. You will also need to install the **beautifulsoup**, **pandas-datareader**, and **wbgapi** packages in your terminal using `uv add beautifulsoup4`, and `uv add wbgapi` respectively. We'll also use two built-in packages, **textwrap** and **requests**.\n", "\n", "To kick off, let's import some of the packages we need (it's always good practice to import the packages you need at the top of a script or notebook)." ] @@ -82,11 +82,10 @@ "import textwrap\n", "\n", "import pandas as pd\n", + "import polars as pl\n", "import requests\n", "from bs4 import BeautifulSoup\n", - "from lets_plot import *\n", - "\n", - "LetsPlot.setup_html()" + "import lets_plot as lp" ] }, { @@ -95,7 +94,7 @@ "id": "f43a5237", "metadata": {}, "source": [ - "## Extracting Data from Files on the Internet using **pandas**\n", + "## Extracting Data from Files on the Internet using **polars**\n", "\n", "It's easy to read data from the internet once you have the url and file type. Here, for instance, is an example that reads in the 'storms' dataset, which is stored as a CSV file in a URL (we'll only grab the first 10 rows):" ] @@ -107,8 +106,8 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_csv(\n", - " \"https://vincentarelbundock.github.io/Rdatasets/csv/dplyr/storms.csv\", nrows=10\n", + "pl.read_csv(\n", + " \"https://vincentarelbundock.github.io/Rdatasets/csv/dplyr/storms.csv\", n_rows=10\n", ")" ] }, @@ -132,13 +131,13 @@ "\n", "An API has an 'endpoint', the base url, and then a URL that encodes the question. Let's see an example with the ONS API for which the endpoint is \"https://api.beta.ons.gov.uk/v1/\". The rest of the API has the form 'data?uri=' and then the long ID of both the timeseries (jp9z) and then the dataset (LMS), which is vacancies in the UK services sector.\n", "\n", - "The data that are returned by APIs are typically in JSON format, which looks a lot like a nested Python dictionary and its entries can be accessed in the same way--this is what is happening when getting the series' title in the example below. JSON is not good for analysis, so we'll use **pandas** to put the data into shape." + "The data that are returned by APIs are typically in JSON format, which looks a lot like a nested Python dictionary and its entries can be accessed in the same way--this is what is happening when getting the series' title in the example below. JSON is not good for analysis, so we'll use **polars** to put the data into shape." ] }, { "cell_type": "code", "execution_count": null, - "id": "c4226d67", + "id": "6107093c", "metadata": {}, "outputs": [], "source": [ @@ -147,57 +146,44 @@ "# Get the data from the ONS API:\n", "json_data = requests.get(url).json()\n", "\n", - "# Prep the data for a quick plot\n", "title = json_data[\"description\"][\"title\"]\n", + "\n", + "# Convert dates using string operations\n", "df = (\n", - " pd.DataFrame(pd.json_normalize(json_data[\"months\"]))\n", - " .assign(\n", - " date=lambda x: pd.to_datetime(x[\"date\"]),\n", - " value=lambda x: pd.to_numeric(x[\"value\"]),\n", - " )\n", - " .set_index(\"date\")\n", + " pl.DataFrame(json_data[\"months\"])\n", + " .with_columns([\n", + " # Add day to make it a valid date string\n", + " (pl.col(\"date\") + \"-01\").str.to_date(format=\"%Y %b-%d\").alias(\"date\"),\n", + " pl.col(\"value\").cast(pl.Float64).alias(\"value\"),\n", + " ])\n", + " .drop_nulls(\"date\")\n", + " .sort(\"date\")\n", ")\n", "\n", - "df[\"value\"].plot(title=title, ylim=(0, df[\"value\"].max() * 1.2), lw=3.0);" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "670ce0bb", - "metadata": {}, - "source": [ - "We've talked about *reading* APIs. You can also create your own to serve up data, models, whatever you like! This is an advanced topic and we won't cover it; but if you do need to, the simplest way is to use [Fast API](https://fastapi.tiangolo.com/). You can find some short video tutorials for Fast API [here](https://calmcode.io/fastapi/hello-world.html).\n", - "\n", - "### Pandas Datareader: an easier way to interact with (some) APIs\n", - "\n", - "Although it didn't take much code to get the ONS data, it would be even better if it was just a single line, wouldn't it? Fortunately there are some packages out there that make this easy, but it does depend on the API (and APIs come and go over time).\n", "\n", - "By far the most comprehensive library for accessing extra APIs is [**pandas-datareader**](https://pandas-datareader.readthedocs.io/en/latest/), which provides convenient access to:\n", "\n", - "- FRED\n", - "- Quandl\n", - "- World Bank\n", - "- OECD\n", - "- Eurostat\n", + "# Initialize the library\n", + "lp.LetsPlot.setup_html()\n", "\n", - "and more.\n", + "# Create plot using the alias\n", + "chart = lp.ggplot(df, lp.aes(x=\"date\", y=\"value\")) + \\\n", + " lp.geom_line(size=2.0, color=\"steelblue\") + \\\n", + " lp.ggtitle(title) + \\\n", + " lp.ylim(0, df[\"value\"].max() * 1.2) + \\\n", + " lp.theme_classic()\n", "\n", - "Let's see an example using FRED (the Federal Reserve Bank of St. Louis' economic data library). This time, let's look at the UK unemployment rate:" + "chart" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "bf758fb4", + "attachments": {}, + "cell_type": "markdown", + "id": "670ce0bb", "metadata": {}, - "outputs": [], "source": [ - "import pandas_datareader.data as web\n", - "\n", - "df_u = web.DataReader(\"LRHUTTTTGBM156S\", \"fred\")\n", + "We've talked about *reading* APIs. You can also create your own to serve up data, models, whatever you like! This is an advanced topic and we won't cover it; but if you do need to, the simplest way is to use [Fast API](https://fastapi.tiangolo.com/). You can find some short video tutorials for Fast API [here](https://calmcode.io/fastapi/hello-world.html).\n", "\n", - "df_u.plot(title=\"UK unemployment (percent)\", legend=False, ylim=(2, 6), lw=3.0);" + "### Accessing World Bank Data with wbgapi" ] }, { @@ -206,7 +192,9 @@ "id": "0613aefb", "metadata": {}, "source": [ - "And, because it's also a really useful one, let's see how to use the [**wbgapi**](https://pypi.org/project/wbgapi/) package to access World Bank data. (**pandas-datareader** used to provide a World Bank reader too, but it has not been actively maintained, so we prefer **wbgapi** for new work.)" + "While APIs can be accessed directly using tools like requests, some specialized libraries make working with structured datasets much easier. One such example is wbgapi, which provides a convenient interface for accessing World Bank data.\n", + "\n", + "Let’s look at an example using World Bank data on CO₂-equivalent emissions per capita:" ] }, { @@ -224,18 +212,23 @@ "import wbgapi as wb\n", "\n", "indicator_code = \"EN.GHG.ALL.PC.CE.AR5\"\n", + "\n", "df = (\n", - " wb.data.DataFrame(\n", - " indicator_code,\n", - " [\"USA\", \"CHN\", \"IND\", \"EAS\", \"ECS\"], # country and region codes\n", - " time=range(2019, 2020),\n", - " labels=True,\n", + " pl.from_pandas(\n", + " wb.data.DataFrame(\n", + " indicator_code,\n", + " [\"USA\", \"CHN\", \"IND\", \"EAS\", \"ECS\"],\n", + " time=range(2019, 2020),\n", + " labels=True,\n", + " ).reset_index()\n", " )\n", - " .rename(columns={\"Country\": \"country\", \"YR2019\": indicator_code})\n", - " .reset_index(drop=True)\n", + " .rename({\"Country\": \"country\"})\n", + " .with_columns(\n", + " pl.col(\"country\").map_elements(lambda x: textwrap.fill(x, 10))\n", + " )\n", + " .sort(indicator_code, descending=True)\n", ")\n", - "df[\"country\"] = df[\"country\"].apply(lambda x: textwrap.fill(x, 10)) # wrap long names\n", - "df = df.sort_values(indicator_code) # re-order\n", + "\n", "df.head()" ] }, @@ -246,19 +239,26 @@ "metadata": {}, "outputs": [], "source": [ - "(\n", - " ggplot(df, aes(x=\"country\", y=indicator_code))\n", - " + geom_bar(aes(fill=\"country\"), color=\"black\", alpha=0.8, stat=\"identity\")\n", - " + scale_fill_discrete()\n", - " + theme_minimal()\n", - " + theme(legend_position=\"none\")\n", - " + ggsize(600, 400)\n", - " + labs(\n", + "lp.LetsPlot.setup_html()\n", + "\n", + "country_order = df[\"country\"].to_list()\n", + "\n", + "plot = (\n", + " lp.ggplot(df, lp.aes(x=\"country\", y=indicator_code))\n", + " + lp.geom_bar(lp.aes(fill=\"country\"), color=\"black\", alpha=0.8, stat=\"identity\")\n", + " + lp.scale_x_discrete(limits=country_order)\n", + " + lp.scale_fill_discrete()\n", + " + lp.theme_minimal()\n", + " + lp.theme(legend_position=\"none\")\n", + " + lp.ggsize(600, 400)\n", + " + lp.labs(\n", " subtitle=\"Greenhouse gases (CO2-equivalent metric tons per capita, 2019)\",\n", " title=\"The USA leads the world on per-capita emissions\",\n", " y=\"\",\n", " )\n", - ")" + ")\n", + "\n", + "plot.show()" ] }, { diff --git a/workflow-help.quarto_ipynb_1 b/workflow-help.quarto_ipynb_1 new file mode 100644 index 0000000..f2d980b --- /dev/null +++ b/workflow-help.quarto_ipynb_1 @@ -0,0 +1,115 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Postscript: Getting Further Help {#sec-workflow-help}\n", + "\n", + "This book is not an island; there is no single resource that will allow you to master Python for Data Science. As you begin to apply the techniques described in this book to your own data, you will soon find questions that we do not answer. This section describes a few tips on how to get help, and to help you keep learning.\n", + "\n", + "## Resources\n", + "\n", + "Some other resources for learning are:\n", + "\n", + "- [The Python Data Science Handbook](https://jakevdp.github.io/PythonDataScienceHandbook/)\n", + "- [Real Python](https://realpython.com/), which has excellent short tutorials that cover Python more broadly (not just data science)\n", + "- [freeCodeCamp's Python courses](https://www.freecodecamp.org/news/search?query=data%20science%20python), though take care to select one that's at the right level for you\n", + "- [Coding for Economists](https://aeturrell.github.io/coding-for-economists), which has similar content to this book but is more in depth and aimed at analysts (particularly in economics)\n", + "\n", + "## Google is your friend\n", + "\n", + "If you get stuck, start with Google. Typically adding \"Python\" or \"Python Data Science\" (as the Python ecosystem goes *well* beyond data science) to a query is enough to restrict it to relevant results. Google is particularly useful for error messages. If you get an error message and you have no idea what it means, try googling it! Chances are that someone else has been confused by it in the past, and there will be help somewhere on the web.\n", + "\n", + "If Google doesn't help, try [Stack Overflow](http://stackoverflow.com). Start by spending a little time searching for an existing answer, including `[Python]` to restrict your search to questions and answers that use Python.\n", + "\n", + "## In the loop\n", + "\n", + "It's also helpful to keep an eye on the latest developments in data science. There are tons of data science newsletters out there, and we recommend keeping up with the Python data science community by following the (#pydata), (#datascience), and (#python) hashtags on Twitter.\n", + "\n", + "## Making a reprex (reproducible example)\n", + "\n", + "If your googling doesn't find anything useful, it's a really good idea prepare a minimal reproducible example or **reprex**.\n", + "\n", + "A good reprex makes it easier for other people to help you, and often you'll figure out the problem yourself in the course of making it. There are two parts to creating a reprex:\n", + "\n", + "- First, you need to make your code reproducible. This means that you need to capture everything, i.e., include any packages you used and create all necessary objects. The easiest way to make sure you've done this is to use the [**watermark**](https://github.com/rasbt/watermark) package alongside whatever else you are doing:" + ], + "id": "8f49fc2e" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import watermark.watermark as watermark\n", + "\n", + "print(watermark())\n", + "print(watermark(iversions=True, globals_=globals()))" + ], + "id": "eca8f7b4", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Second, you need to make it minimal. Strip away everything that is not directly related to your problem. This usually involves creating a much smaller and simpler Python object than the one you're facing in real life or even using built-in data.\n", + "\n", + "That sounds like a lot of work! And it can be, but it has a great payoff:\n", + "\n", + "- 80% of the time creating an excellent reprex reveals the source of your problem. It's amazing how often the process of writing up a self-contained and minimal example allows you to answer your own question.\n", + "\n", + "- The other 20% of time you will have captured the essence of your problem in a way that is easy for others to play with. This substantially improves your chances of getting help.\n", + "\n", + "There are several things you need to include to make your example reproducible: Python environment, required packages, data, and code.\n", + "\n", + "- **Python environment**--really just the Python version. This is covered by the first call to the **watermark** package.\n", + "\n", + "- **Packages** and their versions. These should be loaded at the top of the script, so it's easy to see which ones the example needs. By using **watermark** with the above configuration, you will also print the package versions. This is a good time to check that you're using the latest version of each package; it's possible you've discovered a bug that's been fixed since you installed or last updated the package.\n", + "\n", + "- **Data**: as others won't be able to easily download the data you're working with, it's often best to create a small amount of data from code that still have the same problem as you're finding with your actual data. Between **numpy** and **pandas**, it's quite easy to generate data from code; here's an example:" + ], + "id": "6ffdbbbe" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "df = pd.DataFrame(\n", + " data=np.reshape(range(36), (6, 6)),\n", + " index=[\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"],\n", + " columns=[\"col\" + str(i) for i in range(6)],\n", + " dtype=float,\n", + ")\n", + "df[\"random_normal\"] = np.random.normal(size=6)\n", + "df" + ], + "id": "6b14eb61", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- **Code**: copy and paste the minimal reproducible example code (including the packages, as noted above). Make sure you've used spaces and your variable names are concise, yet informative. Use comments to indicate where your problem lies. Do your best to remove everything that is not related to the problem. Finally, the shorter your code is, the easier it is to understand, and the easier it is to fix.\n", + "\n", + "Finish by checking that you have actually made a reproducible example by starting a fresh Python session and copying and pasting your reprex in." + ], + "id": "154142ef" + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "language": "python", + "display_name": "Python 3 (ipykernel)", + "path": "/Users/omagic/Documents/GitHub/python4DSpolars/.venv/share/jupyter/kernels/python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/workflow-packages-and-environments.quarto_ipynb_1 b/workflow-packages-and-environments.quarto_ipynb_1 new file mode 100644 index 0000000..fda4725 --- /dev/null +++ b/workflow-packages-and-environments.quarto_ipynb_1 @@ -0,0 +1,149 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Workflow: Packages and Environments {#sec-workflow-packages-and-environments}\n", + "\n", + "In this chapter, you're going to learn about packages and how to install them plus virtual coding environments that keep your packages isolated and your projects reproducible.\n", + "\n", + "## Packages\n", + "\n", + "### Introduction\n", + "\n", + "Packages (also called libraries) are key to extending the functionality of Python. It won't be long before you'll need to install some. There are packages for geoscience, for building websites, for analysing genetic data, for economics—pretty much for anything you can think of. Packages are typically not written by the core maintainers of the Python language but by enthusiasts, firms, researchers, academics, all sorts! Because anyone can write packages, they vary widely in their quality and usefulness. There are some that you'll be seeing them again and again.\n", + "\n", + "

Name a more iconic trio, I'll wait. pic.twitter.com/pGaLuUxQ3r

— Vicki Boykis (\\@vboykis) August 23, 2018
\n", + "\n", + "The three Python packages **numpy**, **pandas**, and **maplotlib**, which respectively provide numerical, data analysis, and plotting functionality, are ubiquitous. So many scripts begin by importing all three of them, as in the tweet above!\n", + "\n", + "There are typically two steps to using a new Python package:\n", + "\n", + "1. *install* the package on the command line (aka the terminal), eg using `uv add pandas`\n", + "\n", + "2. *import* the package into your Python session, eg using `import pandas as pd`\n", + "\n", + "When you issue an install command for a specific package, it is automatically downloaded from the internet and installed in the appropriate place on your computer. To install extra Python packages, you issue install commands to a text-based window called the \"terminal\".\n", + "\n", + "### The Command Line in Brief\n", + "\n", + "The *terminal* or *command line* or sometimes the *command prompt* was labelled 4 in the screenshot of Visual Studio Code from the chapter on @sec-introduction. The terminal is a text-based way to issue all kinds of commands to your computer (not just Python commands) and knowing a little bit about it is really useful for coding (and more) because managing packages, environments (which we haven't yet discussed), and version control (ditto) can all be done via the terminal. We'll come to these in due course in the chapter on @sec-command-line, but for now, a little background on what the terminal is and what it does.\n", + "\n", + "::: {.callout-note}\n", + "To open up the command line within Visual Studio Code, use the ⌃ + \\` keyboard shortcut (Mac) or ctrl + \\` (Windows/Linux), or click \"View > Terminal\".\n", + "\n", + "If you want to open up the command line independently of Visual Studio Code, search for \"Terminal\" on Mac and Linux, and \"Powershell\" on Windows.\n", + ":::\n", + "\n", + "Firstly, everything you can do by clicking on icons to launch programmes on your computer, you can also do via the terminal, also known as the command line. For many programmes, a lot of their functionality can be accessed using the command line, and other programmes *only* have a command line interface (CLI), including some that are used for data science.\n", + "\n", + "::: {.callout-tip}\n", + "The command line interacts with your operating system and is used to create, activate, or change Python installations.\n", + ":::\n", + "\n", + "Use Visual Studio Code to open a terminal window by clicking Terminal -> New Terminal on the list of commands at the very top of the window. If you have installed uv on your computer, your terminal should look something like this as your 'command prompt':\n", + "\n", + "```bash\n", + "your-username@your-computer current-directory %\n", + "```\n", + "\n", + "on Mac, and the same but with '%' replaced by '$' on linux, and (using Powershell)\n", + "\n", + "```powershell\n", + "PS C:\\Windows\\System32>\n", + "```\n", + "\n", + "on Windows.\n", + "\n", + "You can check that uv has successfully installed Python in your current project's folder by running\n", + "\n", + "```bash\n", + "uv run python --version\n", + "```\n", + "\n", + "For now, to at least try out the command line, let's use something that works across all three of the major operating systems. Type `uv run python` on the command prompt that came up in your new terminal window. You should see information about your installation of Python appear, including the version, followed by a Python prompt that looks like `>>>`. This is a kind of interactive Python session, in the terminal. It's much less rich than the one available in Visual Studio Code (it can't run scripts line-by-line, for example) but you can try `print('Hello World!')` and it will run, printing your message. To exit the terminal-based Python session, type `exit()` to go back to the regular command line.\n", + "\n", + "### Installing Packages\n", + "\n", + "To install extra Python packages, the default and easiest way is to use `uv add **packagename**`. There are over 330,000 Python packages on PyPI (the Python Package Index)! You can see what packages you have installed already by running `uv pip list` into the command line.\n", + "\n", + "`uv add ...` will install packages into the special Python environment in your current folder (it sits in a subdirectory called \".venv\" which will be hidden by default on most systems.) It's really helpful and good practice to have one Python environment per project, and **uv** does this automatically for you.\n", + "\n", + "::: {.callout-tip title=\"Exercise\"}\n", + "Try installing the **matplotlib**, **pandas**, **statsmodels**, and **skimpy** packages using `uv add`.\n", + ":::\n", + "\n", + "### Using Packages\n", + "\n", + "Once you have installed a package, you need to be able to use it! This is usually done via an import statement at the top of your script or Jupyter Notebook. For example, to bring in **pandas**, it's\n", + "\n", + "```python\n", + "import pandas as pd\n", + "```\n", + "\n", + "Why does Python do this? The idea of not just loading every package is to provide clarity over what function is being called from what package. It's also not necessary to load every package for every piece of analysis, and you often actually want to know what the *minimum* set of packages is to reproduce an analysis. Making the package imports explicit helps with all of that.\n", + "\n", + "You may also wonder why one doesn't just use `import pandas as pandas`. There's actually nothing stopping you doing this except i) it's convenient to have a shorter name and ii) there does tend to be a convention around imports, ie `pd` for **pandas** and `np` for **numpy**, and your code will be clearer to yourself and others if you follow the conventions.\n", + "\n", + "## Virtual Code Environments\n", + "\n", + "Virtual code environments allow you to isolate all of the packages that you're using to do analysis for one project from the set of packages you might need for a different project. They're an important part of creating a reproducible analytical pipeline but a key benefit is that others can reproduce the environment you used and it's best practice to have an isolated environment per project.\n", + "\n", + "To be more concrete, let's say you're using Python 3.9, **statsmodels**, and **pandas** for one project, project A. And, for project B, you need to use Python 3.10 with **numpy** and **scikit-learn**. Even with the same version of Python, best practice would be to have two separate virtual Python environments: environment A, with everything needed for project A, and environment B, with everything needed for project B. For the case where you're using different versions of Python, this isn't just best practice, it's essential.\n", + "\n", + "Many programming languages now come with an option to install packages and a version of the language in isolated environments. In Python, there are multiple tools for managing different environments. And, of those, the easiest to work with is probably [**uv**](https://docs.astral.sh/uv/).\n", + "\n", + "You can see all of the packages in the environment created in your current folder by running `uv pip list` on the command line. Here's an example of looking at the installed packages within this very book, filtering them just to the ones beginning with \"s\".\n", + "\n", + "```{bash}\n", + "uv run pip list | grep ^s\n", + "```\n", + "\n", + "### The pyproject.toml file in Python Environments\n", + "\n", + "You may have noticed that a file called `pyproject.toml` has been created." + ], + "id": "6b7f7364" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import toml\n", + "from rich import print_json\n", + "\n", + "print_json(data=toml.load(\"pyproject.toml\"))" + ], + "id": "acc9e141", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This lists all of the dependencies, and the version, of a **uv** Python project. There are lots of benefits to tracking what versions of packages you're using like this. One of the most important is that you can *share* projects with other people, and they can install them from these files too.\n", + "\n", + "As you install or remove packages, the `pyproject.toml` file changes in lockstep.\n", + "\n", + "Noe that Visual Studio Code shows which Python environment you are using when you open a Python script or Jupyter Notebook.\n", + "\n", + "![A typical user view in Visual Studio Code](https://github.com/aeturrell/coding-for-economists/blob/main/img/vscode_layout.png?raw=true)\n", + "\n", + "In the screenshot above, you can see the project-environment in two places: on the blue bar at the bottom of the screen, and (in 5), at the top right hand side of the interactive window. A similar top right indicator is present when you have a Jupyter Notebook open too." + ], + "id": "6ac222b7" + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "language": "python", + "display_name": "Python 3 (ipykernel)", + "path": "/Users/omagic/Documents/GitHub/python4DSpolars/.venv/share/jupyter/kernels/python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file From da6546bcd458d49db4b0e1377c3c19c408afa926 Mon Sep 17 00:00:00 2001 From: Nwabueze Ugoh Date: Wed, 22 Apr 2026 19:49:47 +0100 Subject: [PATCH 22/34] Remove obsolete Quarto notebooks: visualise.quarto_ipynb_1, workflow-help.quarto_ipynb_1, and workflow-packages-and-environments.quarto_ipynb_1. --- visualise.quarto_ipynb_1 | 6 +- webscraping-and-apis.ipynb | 86 ++++++++++++------- workflow-help.quarto_ipynb_1 | 10 +-- ...w-packages-and-environments.quarto_ipynb_1 | 6 +- 4 files changed, 66 insertions(+), 42 deletions(-) diff --git a/visualise.quarto_ipynb_1 b/visualise.quarto_ipynb_1 index eb958d5..2ef6d2f 100644 --- a/visualise.quarto_ipynb_1 +++ b/visualise.quarto_ipynb_1 @@ -28,7 +28,7 @@ "\n", "The first of the three kinds is *exploratory data visualisation*, and it's the kind that you do when you're looking and data and trying to understand it. Just plotting the data is a really good strategy for getting a feel for any issues there might be. This is perhaps most famously demonstrated by Anscombe's quartet: four different datasets with the same mean, standard deviation, and correlation but very different data distributions." ], - "id": "2d4a4a09" + "id": "f3331573" }, { "cell_type": "code", @@ -96,7 +96,7 @@ "plt.suptitle(\"Anscombe's Quartet\")\n", "plt.show()" ], - "id": "c38c38fa", + "id": "64a0e7f6", "execution_count": null, "outputs": [] }, @@ -120,7 +120,7 @@ "\n", "You can find more information on the topic of communicating via data visualisations in the @sec-communicate-plots chapter." ], - "id": "88d12184" + "id": "30b9ff30" } ], "metadata": { diff --git a/webscraping-and-apis.ipynb b/webscraping-and-apis.ipynb index 6e8daf7..fc73aa9 100644 --- a/webscraping-and-apis.ipynb +++ b/webscraping-and-apis.ipynb @@ -267,15 +267,19 @@ "id": "b7bf16d7", "metadata": {}, "source": [ - "### The OECD API\n", + "### The Eurostat SDMX API\n", "\n", - "Sometimes it's convenient to use APIs directly, and, as an example, the OECD API comes with a LOT of complexity that direct access can take advantage of. The OECD API makes data available in both JSON and XML formats, and we'll use [**pandasdmx**](https://pandasdmx.readthedocs.io/) (aka the Statistical Data and Metadata eXchange (SDMX) package for the Python data ecosystem) to pull down the XML format data and turn it into a regular **pandas** data frame.\n", + "Sometimes it’s convenient to use APIs directly. The Eurostat API provides access to a massive repository of European statistical data using the SDMX (Statistical Data and Metadata eXchange) standard. While Eurostat offers multiple formats, using the SDMX-ML (XML) format via the sdmx1 library allows us to pull structured data into the Python ecosystem with high precision.\n", "\n", - "Now, key to using the OECD API is knowledge of its many codes: for countries, times, resources, and series. You can find some broad guidance on what codes the API uses [here](https://data.oecd.org/api/sdmx-ml-documentation/) but to find exactly what you need can be a bit tricky. Two tips are:\n", - "1. If you know what you're looking for is in a particular named dataset, eg \"QNA\" (Quarterly National Accounts), put `https://stats.oecd.org/restsdmx/sdmx.ashx/GetDataStructure/QNA/all?format=SDMX-ML` into your browser and look through the XML file; you can pick out the sub-codes and the countries that are available.\n", - "2. Browse around on https://stats.oecd.org/ and use Customise then check all the \"Use Codes\" boxes to see whatever your browsing's code names.\n", + "Key to using the Eurostat API is understanding the Data Structure Definition (DSD). Every dataset is essentially a multidimensional \"cube\" where each dimension (like Geography, Unit, or Frequency) has specific codes.\n", "\n", - "Let's see an example of this in action. We'd like to see the productivity (GDP per hour) data for a range of countries since 2010. We are going to be in the productivity resource (code \"PDB_LV\") and we want the USD current prices (code \"CPC\") measure of GDP per employed worker (code \"T_GDPEMP) from 2010 onwards (code \"startTime=2010\"). We'll grab this for some developed countries where productivity measurements might be slightly more comparable. The comments below explain what's happening in each step." + "To find the exact codes you need:\n", + "\n", + "The Data Browser: Browse the Eurostat Data Navigation Tree. Once you find a table (e.g., \"HICP - monthly data\"), the \"Dataset Code\" (like prc_hicp_manr) is shown in brackets.\n", + "\n", + "Positional Keys: Eurostat's REST API expects a \"key string\" where codes are placed in a specific order separated by dots (e.g., Freq.Unit.Item.Geo). If you know the order, you can \"slice\" the data cube directly.\n", + "\n", + "Let’s see an example of this in action. We want to see the Harmonised Index of Consumer Prices (HICP)—specifically the annual rate of change for all items—for Germany and France. We will use the resource prc_hicp_manr, requesting Monthly frequency (M), the Annual Rate of Change unit (RCH_A), and the \"All-items\" classification (CP00)." ] }, { @@ -285,17 +289,32 @@ "metadata": {}, "source": [ "```python\n", - "import pandasdmx as pdmx\n", - "# Tell pdmx we want OECD data\n", - "oecd = pdmx.Request(\"OECD\")\n", - "# Set out everything about the request in the format specified by the OECD API\n", - "data = oecd.data(\n", - " resource_id=\"PDB_LV\",\n", - " key=\"GBR+FRA+CAN+ITA+DEU+JPN+USA.T_GDPEMP.CPC/all?startTime=2010\",\n", - ").to_pandas()\n", - "\n", - "df = pd.DataFrame(data).reset_index()\n", - "df.head()\n", + "import sdmx\n", + "import polars as pl\n", + "\n", + "# Tell sdmx we want ESTAT data\n", + "client = sdmx.Client('ESTAT')\n", + "\n", + "# 2. Build the URL-style positional key\n", + "# Format: [Freq].[Unit].[Coicop].[Geo]\n", + "# We use '+' to join multiple countries (DE and FR)\n", + "resource_id = 'prc_hicp_manr'\n", + "key_string = 'M.RCH_A.CP00.DE+FR'\n", + "\n", + "# 3. Fetch the data directly\n", + "# 'startPeriod' limits the timeline to recent data\n", + "response = client.data(\n", + " resource_id=resource_id,\n", + " key=key_string,\n", + " params={'startPeriod': '2024-01'}\n", + ")\n", + "\n", + "# 4. Convert the SDMX-ML response to a Polars DataFrame\n", + "# We bridge through Pandas as sdmx1 is optimized for it\n", + "df_pd = sdmx.to_pandas(response).to_frame(name='value').reset_index()\n", + "df = pl.from_pandas(df_pd)\n", + "\n", + "print(df.head())\n", "```" ] }, @@ -305,13 +324,13 @@ "id": "e5cac233", "metadata": {}, "source": [ - "| | LOCATION | SUBJECT | MEASURE | TIME_PERIOD | value |\n", - "|--:|---------:|---------:|--------:|------------:|-------------:|\n", - "| 0 | CAN | T_GDPEMP | CPC | 2010 | 78848.604088 |\n", - "| 1 | CAN | T_GDPEMP | CPC | 2011 | 81422.364748 |\n", - "| 2 | CAN | T_GDPEMP | CPC | 2012 | 82663.028058 |\n", - "| 3 | CAN | T_GDPEMP | CPC | 2013 | 86368.582158 |\n", - "| 4 | CAN | T_GDPEMP | CPC | 2014 | 89617.632446 |" + "| | TIME_PERIOD | geo | unit | freq | coicop | value |\n", + "|--:|------------:|:----|:------|:-----|:-------|------:|\n", + "| 0 | 2024-01 | DE | RCH_A | M | CP00 | 3.1 |\n", + "| 1 | 2024-02 | DE | RCH_A | M | CP00 | 2.7 |\n", + "| 2 | 2024-03 | DE | RCH_A | M | CP00 | 2.3 |\n", + "| 3 | 2024-04 | DE | RCH_A | M | CP00 | 2.4 |\n", + "| 4 | 2024-05 | DE | RCH_A | M | CP00 | 2.8 |" ] }, { @@ -489,11 +508,11 @@ "source": [ "### Webscraping Tables\n", "\n", - "Often there are times when you don't actually want to scrape an entire webpage and all you want is the data from a *table* within the page. Fortunately, there is an easy way to scrape individual tables using the **pandas** package.\n", + "There are times when you don't need to scrape an entire webpage; you simply want the structured data from a specific table. While Polars is a high-performance data engine, it focuses on strict data formats (like Parquet or CSV) and does not natively include an HTML parser. However, we can easily bridge this gap by using Pandas to fetch the table and then converting it into a Polars DataFrame.\n", "\n", - "We will read data from a table on 'https://webscraper.io/test-sites/tables' using **pandas**. The function we'll use is `read_html()`, which returns a list of data frames of all the tables it finds when you pass it a URL. If you want to filter the list of tables, use the `match=` keyword argument with text that only appears in the table(s) you're interested in.\n", + "We will read data from 'https://webscraper.io/test-sites/tables' using `pd.read_html()`. This function scans the webpage and returns a list of all tables it finds as DataFrames. To target a specific table, we use the match= keyword argument with text that uniquely appears in the table we want—in this case, \"First Name\".\n", "\n", - "The example below shows how this works; looking at the website, we can see that the table we're interested in, has a 'First Name' column. Therefore we run:" + "Once captured, we convert the result to Polars using pl.from_pandas() to take advantage of Polars' superior query performance and expression API." ] }, { @@ -503,10 +522,15 @@ "metadata": {}, "outputs": [], "source": [ - "df_list = pd.read_html(\"https://webscraper.io/test-sites/tables\", match=\"First Name\")\n", + "import polars as pl\n", + "import pandas as pd\n", + "\n", + "\n", + "pd_list = pd.read_html(\"https://webscraper.io/test-sites/tables\", match=\"First Name\")\n", "# Retrieve first entry from list of data frames\n", - "df = df_list[0]\n", - "df.head()" + "df = pl.from_pandas(pd_list[0])\n", + "\n", + "print(df.head())" ] }, { @@ -515,7 +539,7 @@ "id": "31e49317", "metadata": {}, "source": [ - "This gives us the table neatly loaded into a **pandas** data frame ready for further use.\n", + "This gives us the table neatly loaded into a **polars** data frame ready for further use.\n", "\n", "If you get a '403' error, it means that the website has blocked **pandas** because it can see that you are engaged in web scraping. This is because some people web scrape irresponsibly, or because websites have provided other, preferred ways for you to obtain the data, eg via a download of the whole thing (think Wikipedia) or through an API. (If you really need to, [you can often get around the 403 error](https://stackoverflow.com/questions/43590153/http-error-403-forbidden-when-reading-html) though.)" ] diff --git a/workflow-help.quarto_ipynb_1 b/workflow-help.quarto_ipynb_1 index f2d980b..e7bebf8 100644 --- a/workflow-help.quarto_ipynb_1 +++ b/workflow-help.quarto_ipynb_1 @@ -35,7 +35,7 @@ "\n", "- First, you need to make your code reproducible. This means that you need to capture everything, i.e., include any packages you used and create all necessary objects. The easiest way to make sure you've done this is to use the [**watermark**](https://github.com/rasbt/watermark) package alongside whatever else you are doing:" ], - "id": "8f49fc2e" + "id": "22b3f9e0" }, { "cell_type": "code", @@ -48,7 +48,7 @@ "print(watermark())\n", "print(watermark(iversions=True, globals_=globals()))" ], - "id": "eca8f7b4", + "id": "a119501b", "execution_count": null, "outputs": [] }, @@ -72,7 +72,7 @@ "\n", "- **Data**: as others won't be able to easily download the data you're working with, it's often best to create a small amount of data from code that still have the same problem as you're finding with your actual data. Between **numpy** and **pandas**, it's quite easy to generate data from code; here's an example:" ], - "id": "6ffdbbbe" + "id": "c4ac60b4" }, { "cell_type": "code", @@ -87,7 +87,7 @@ "df[\"random_normal\"] = np.random.normal(size=6)\n", "df" ], - "id": "6b14eb61", + "id": "d1e4562c", "execution_count": null, "outputs": [] }, @@ -99,7 +99,7 @@ "\n", "Finish by checking that you have actually made a reproducible example by starting a fresh Python session and copying and pasting your reprex in." ], - "id": "154142ef" + "id": "4b75e409" } ], "metadata": { diff --git a/workflow-packages-and-environments.quarto_ipynb_1 b/workflow-packages-and-environments.quarto_ipynb_1 index fda4725..a5600ce 100644 --- a/workflow-packages-and-environments.quarto_ipynb_1 +++ b/workflow-packages-and-environments.quarto_ipynb_1 @@ -104,7 +104,7 @@ "\n", "You may have noticed that a file called `pyproject.toml` has been created." ], - "id": "6b7f7364" + "id": "8b889898" }, { "cell_type": "code", @@ -115,7 +115,7 @@ "\n", "print_json(data=toml.load(\"pyproject.toml\"))" ], - "id": "acc9e141", + "id": "688f09f1", "execution_count": null, "outputs": [] }, @@ -133,7 +133,7 @@ "\n", "In the screenshot above, you can see the project-environment in two places: on the blue bar at the bottom of the screen, and (in 5), at the top right hand side of the interactive window. A similar top right indicator is present when you have a Jupyter Notebook open too." ], - "id": "6ac222b7" + "id": "148595b3" } ], "metadata": { From 11062ba1f2844b36fb4f2f539d6ea9dda61f4f1b Mon Sep 17 00:00:00 2001 From: Nwabueze Ugoh Date: Thu, 23 Apr 2026 23:29:51 +0100 Subject: [PATCH 23/34] Remove obsolete Quarto notebooks: visualise.quarto_ipynb_1, workflow-help.quarto_ipynb_1, and workflow-packages-and-environments.quarto_ipynb_1. --- visualise.quarto_ipynb_1 | 136 ---------------- workflow-help.quarto_ipynb_1 | 115 -------------- ...w-packages-and-environments.quarto_ipynb_1 | 149 ------------------ 3 files changed, 400 deletions(-) delete mode 100644 visualise.quarto_ipynb_1 delete mode 100644 workflow-help.quarto_ipynb_1 delete mode 100644 workflow-packages-and-environments.quarto_ipynb_1 diff --git a/visualise.quarto_ipynb_1 b/visualise.quarto_ipynb_1 deleted file mode 100644 index 2ef6d2f..0000000 --- a/visualise.quarto_ipynb_1 +++ /dev/null @@ -1,136 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Visualisation {#sec-visualise}\n", - "\n", - "After reading the first part of the book, you understand the basics of the most important tools for doing data science. Now it’s time to start diving into the details. In this part of the book, you’ll learn about visualising data in further depth (in @sec-vis-layers), and get further stuck into the details of the different kinds of data visualisation (in @sec-exploratory-data-analysis and @sec-communicate-plots). In this short chapter, we discuss the different ways to create visualisations, and the different purposes of visualisations.\n", - "\n", - "## Philosophies of data visualisation\n", - "\n", - "There are broadly two categories of approach to using code to create data visualisations: *imperative* (build what you want from individual elements) and *declarative* (say what you want from a list of pre-existing options). Choosing which to use involves a trade-off: imperative libraries offer you flexibility but at the cost of some verbosity; declarative libraries offer you a quick way to plot your data, but only if it’s in the right format to begin with, and customisation to special chart types is more difficult.\n", - "\n", - "Python has many excellent plotting packages, including perhaps the most powerful imperative plotting package around, **matplotlib**, and an amazing declarative library that we already saw, **lets-plot**. These two libraries will get you a long way, and each could be worthy of an entire book themselves. Fortunately for us, though, we can do 95% of what we need with a small number of commands from one or the other of them. In general, to keep this book as light as possible, we've opted to use **lets-plot** wherever possible—and @sec-vis-layers is going to take you on a more in-depth tour of how to use it yourself.\n", - "\n", - "## Purposes of data visualisation\n", - "\n", - "Data visualisation has all kinds of different purposes. It can be useful to bear in mind three broad categories of visualisation that are out there:\n", - "\n", - "- exploratory\n", - "- scientific\n", - "- narrative\n", - "\n", - "Let's look at each in a bit more detail.\n", - "\n", - "### Exploratory Data Viz\n", - "\n", - "The first of the three kinds is *exploratory data visualisation*, and it's the kind that you do when you're looking and data and trying to understand it. Just plotting the data is a really good strategy for getting a feel for any issues there might be. This is perhaps most famously demonstrated by Anscombe's quartet: four different datasets with the same mean, standard deviation, and correlation but very different data distributions." - ], - "id": "f3331573" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "#| echo: false\n", - "import numpy as np\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib_inline.backend_inline\n", - "\n", - "# Plot settings\n", - "plt.style.use(\"https://github.com/aeturrell/python4DS/raw/main/plot_style.txt\")\n", - "matplotlib_inline.backend_inline.set_matplotlib_formats(\"svg\")\n", - "\n", - "# Set max rows displayed for readability\n", - "pd.set_option(\"display.max_rows\", 6)\n", - "\n", - "x = [10, 8, 13, 9, 11, 14, 6, 4, 12, 7, 5]\n", - "y1 = [8.04, 6.95, 7.58, 8.81, 8.33, 9.96, 7.24, 4.26, 10.84, 4.82, 5.68]\n", - "y2 = [9.14, 8.14, 8.74, 8.77, 9.26, 8.10, 6.13, 3.10, 9.13, 7.26, 4.74]\n", - "y3 = [7.46, 6.77, 12.74, 7.11, 7.81, 8.84, 6.08, 5.39, 8.15, 6.42, 5.73]\n", - "x4 = [8, 8, 8, 8, 8, 8, 8, 19, 8, 8, 8]\n", - "y4 = [6.58, 5.76, 7.71, 8.84, 8.47, 7.04, 5.25, 12.50, 5.56, 7.91, 6.89]\n", - "\n", - "datasets = {\"I\": (x, y1), \"II\": (x, y2), \"III\": (x, y3), \"IV\": (x4, y4)}\n", - "\n", - "fig, axs = plt.subplots(\n", - " 2,\n", - " 2,\n", - " sharex=True,\n", - " sharey=True,\n", - " figsize=(10, 6),\n", - " gridspec_kw={\"wspace\": 0.08, \"hspace\": 0.08},\n", - ")\n", - "axs[0, 0].set(xlim=(0, 20), ylim=(2, 14))\n", - "axs[0, 0].set(xticks=(0, 10, 20), yticks=(4, 8, 12))\n", - "\n", - "for ax, (label, (x, y)) in zip(axs.flat, datasets.items()):\n", - " ax.text(0.1, 0.9, label, fontsize=20, transform=ax.transAxes, va=\"top\")\n", - " ax.tick_params(direction=\"in\", top=True, right=True)\n", - " ax.plot(x, y, \"o\")\n", - "\n", - " # linear regression\n", - " p1, p0 = np.polyfit(x, y, deg=1) # slope, intercept\n", - " ax.axline(xy1=(0, p0), slope=p1, color=\"r\", lw=2)\n", - "\n", - " # add text box for the statistics\n", - " stats = (\n", - " f\"$\\\\mu$ = {np.mean(y):.2f}\\n\"\n", - " f\"$\\\\sigma$ = {np.std(y):.2f}\\n\"\n", - " f\"$r$ = {np.corrcoef(x, y)[0][1]:.2f}\"\n", - " )\n", - " bbox = dict(boxstyle=\"round\", fc=\"blanchedalmond\", ec=\"orange\", alpha=0.5)\n", - " ax.text(\n", - " 0.95,\n", - " 0.07,\n", - " stats,\n", - " fontsize=9,\n", - " bbox=bbox,\n", - " transform=ax.transAxes,\n", - " horizontalalignment=\"right\",\n", - " )\n", - "\n", - "plt.suptitle(\"Anscombe's Quartet\")\n", - "plt.show()" - ], - "id": "64a0e7f6", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Exploratory visualisation is usually quick and dirty, and flexible too. Some exploratory data viz can be automated, and there's a whole host of packages to help with this, including [**skimpy**](https://aeturrell.github.io/skimpy/).\n", - "\n", - "Beyond you and perhaps your co-authors/collaborators, however, not many other people should be seeing your exploratory visualisation! They will typically be worked up quickly, be numerous, and be throw-away. We'll look more at this in @sec-exploratory-data-analysis.\n", - "\n", - "### Scientific Data Viz\n", - "\n", - "The second kind, scientific data visualisation, is the prime cut of your exploratory visualisation. It's the kind of plot you might include in a more technical paper, the picture that says a thousand words. I often think of the first image of a black hole @akiyama2019first as a prime example of this. You can get away with having a high density of information in a scientific plot and, in short format journals, you may need to. The journal Physical Review Letters, which has an 8 page limit, has a classic of this genre in more or less every issue. Ensuring that important values can be accurately read from the plot is especially important in these kinds of charts. But they can also be the kind of plot that presents the killer results in a study; they might not be exciting to people who don't look at charts for a living, but they might be exciting and, just as importantly, understandable by your peers.\n", - "\n", - "This type of visualisation is especially popular in the big science journals like *Nature* and *Science*, where space is at a premium. We won't cover this type of plot in this book, because it tends to be very bespoke.\n", - "\n", - "### Narrative Data Viz\n", - "\n", - "The third and final kind is narrative data visualisation. This is the one that requires the most thought in the step where you go from the first view to the end product. It's a visualisation that doesn't just show a picture, but gives an insight. These are the kind of visualisations that you might see in the *Financial Times*, *The Economist*, or on the *BBC News* website. They come with aids that help the viewer focus on the aspects that the creator wanted them to (you can think of these aids or focuses as doing for visualisation what bold font does for text). They're well worth using in your work, especially if you're trying to communicate a particular narrative, and especially if the people you're communicating with don't have deep knowledge of the topic. You might use them in a paper that you hope will have a wide readership, in a blog post summarising your work, or in a report intended for a policymaker.\n", - "\n", - "You can find more information on the topic of communicating via data visualisations in the @sec-communicate-plots chapter." - ], - "id": "30b9ff30" - } - ], - "metadata": { - "kernelspec": { - "name": "python3", - "language": "python", - "display_name": "Python 3 (ipykernel)", - "path": "/Users/omagic/Documents/GitHub/python4DSpolars/.venv/share/jupyter/kernels/python3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/workflow-help.quarto_ipynb_1 b/workflow-help.quarto_ipynb_1 deleted file mode 100644 index e7bebf8..0000000 --- a/workflow-help.quarto_ipynb_1 +++ /dev/null @@ -1,115 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Postscript: Getting Further Help {#sec-workflow-help}\n", - "\n", - "This book is not an island; there is no single resource that will allow you to master Python for Data Science. As you begin to apply the techniques described in this book to your own data, you will soon find questions that we do not answer. This section describes a few tips on how to get help, and to help you keep learning.\n", - "\n", - "## Resources\n", - "\n", - "Some other resources for learning are:\n", - "\n", - "- [The Python Data Science Handbook](https://jakevdp.github.io/PythonDataScienceHandbook/)\n", - "- [Real Python](https://realpython.com/), which has excellent short tutorials that cover Python more broadly (not just data science)\n", - "- [freeCodeCamp's Python courses](https://www.freecodecamp.org/news/search?query=data%20science%20python), though take care to select one that's at the right level for you\n", - "- [Coding for Economists](https://aeturrell.github.io/coding-for-economists), which has similar content to this book but is more in depth and aimed at analysts (particularly in economics)\n", - "\n", - "## Google is your friend\n", - "\n", - "If you get stuck, start with Google. Typically adding \"Python\" or \"Python Data Science\" (as the Python ecosystem goes *well* beyond data science) to a query is enough to restrict it to relevant results. Google is particularly useful for error messages. If you get an error message and you have no idea what it means, try googling it! Chances are that someone else has been confused by it in the past, and there will be help somewhere on the web.\n", - "\n", - "If Google doesn't help, try [Stack Overflow](http://stackoverflow.com). Start by spending a little time searching for an existing answer, including `[Python]` to restrict your search to questions and answers that use Python.\n", - "\n", - "## In the loop\n", - "\n", - "It's also helpful to keep an eye on the latest developments in data science. There are tons of data science newsletters out there, and we recommend keeping up with the Python data science community by following the (#pydata), (#datascience), and (#python) hashtags on Twitter.\n", - "\n", - "## Making a reprex (reproducible example)\n", - "\n", - "If your googling doesn't find anything useful, it's a really good idea prepare a minimal reproducible example or **reprex**.\n", - "\n", - "A good reprex makes it easier for other people to help you, and often you'll figure out the problem yourself in the course of making it. There are two parts to creating a reprex:\n", - "\n", - "- First, you need to make your code reproducible. This means that you need to capture everything, i.e., include any packages you used and create all necessary objects. The easiest way to make sure you've done this is to use the [**watermark**](https://github.com/rasbt/watermark) package alongside whatever else you are doing:" - ], - "id": "22b3f9e0" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "import watermark.watermark as watermark\n", - "\n", - "print(watermark())\n", - "print(watermark(iversions=True, globals_=globals()))" - ], - "id": "a119501b", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- Second, you need to make it minimal. Strip away everything that is not directly related to your problem. This usually involves creating a much smaller and simpler Python object than the one you're facing in real life or even using built-in data.\n", - "\n", - "That sounds like a lot of work! And it can be, but it has a great payoff:\n", - "\n", - "- 80% of the time creating an excellent reprex reveals the source of your problem. It's amazing how often the process of writing up a self-contained and minimal example allows you to answer your own question.\n", - "\n", - "- The other 20% of time you will have captured the essence of your problem in a way that is easy for others to play with. This substantially improves your chances of getting help.\n", - "\n", - "There are several things you need to include to make your example reproducible: Python environment, required packages, data, and code.\n", - "\n", - "- **Python environment**--really just the Python version. This is covered by the first call to the **watermark** package.\n", - "\n", - "- **Packages** and their versions. These should be loaded at the top of the script, so it's easy to see which ones the example needs. By using **watermark** with the above configuration, you will also print the package versions. This is a good time to check that you're using the latest version of each package; it's possible you've discovered a bug that's been fixed since you installed or last updated the package.\n", - "\n", - "- **Data**: as others won't be able to easily download the data you're working with, it's often best to create a small amount of data from code that still have the same problem as you're finding with your actual data. Between **numpy** and **pandas**, it's quite easy to generate data from code; here's an example:" - ], - "id": "c4ac60b4" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "df = pd.DataFrame(\n", - " data=np.reshape(range(36), (6, 6)),\n", - " index=[\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"],\n", - " columns=[\"col\" + str(i) for i in range(6)],\n", - " dtype=float,\n", - ")\n", - "df[\"random_normal\"] = np.random.normal(size=6)\n", - "df" - ], - "id": "d1e4562c", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- **Code**: copy and paste the minimal reproducible example code (including the packages, as noted above). Make sure you've used spaces and your variable names are concise, yet informative. Use comments to indicate where your problem lies. Do your best to remove everything that is not related to the problem. Finally, the shorter your code is, the easier it is to understand, and the easier it is to fix.\n", - "\n", - "Finish by checking that you have actually made a reproducible example by starting a fresh Python session and copying and pasting your reprex in." - ], - "id": "4b75e409" - } - ], - "metadata": { - "kernelspec": { - "name": "python3", - "language": "python", - "display_name": "Python 3 (ipykernel)", - "path": "/Users/omagic/Documents/GitHub/python4DSpolars/.venv/share/jupyter/kernels/python3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/workflow-packages-and-environments.quarto_ipynb_1 b/workflow-packages-and-environments.quarto_ipynb_1 deleted file mode 100644 index a5600ce..0000000 --- a/workflow-packages-and-environments.quarto_ipynb_1 +++ /dev/null @@ -1,149 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Workflow: Packages and Environments {#sec-workflow-packages-and-environments}\n", - "\n", - "In this chapter, you're going to learn about packages and how to install them plus virtual coding environments that keep your packages isolated and your projects reproducible.\n", - "\n", - "## Packages\n", - "\n", - "### Introduction\n", - "\n", - "Packages (also called libraries) are key to extending the functionality of Python. It won't be long before you'll need to install some. There are packages for geoscience, for building websites, for analysing genetic data, for economics—pretty much for anything you can think of. Packages are typically not written by the core maintainers of the Python language but by enthusiasts, firms, researchers, academics, all sorts! Because anyone can write packages, they vary widely in their quality and usefulness. There are some that you'll be seeing them again and again.\n", - "\n", - "

Name a more iconic trio, I'll wait. pic.twitter.com/pGaLuUxQ3r

— Vicki Boykis (\\@vboykis) August 23, 2018
\n", - "\n", - "The three Python packages **numpy**, **pandas**, and **maplotlib**, which respectively provide numerical, data analysis, and plotting functionality, are ubiquitous. So many scripts begin by importing all three of them, as in the tweet above!\n", - "\n", - "There are typically two steps to using a new Python package:\n", - "\n", - "1. *install* the package on the command line (aka the terminal), eg using `uv add pandas`\n", - "\n", - "2. *import* the package into your Python session, eg using `import pandas as pd`\n", - "\n", - "When you issue an install command for a specific package, it is automatically downloaded from the internet and installed in the appropriate place on your computer. To install extra Python packages, you issue install commands to a text-based window called the \"terminal\".\n", - "\n", - "### The Command Line in Brief\n", - "\n", - "The *terminal* or *command line* or sometimes the *command prompt* was labelled 4 in the screenshot of Visual Studio Code from the chapter on @sec-introduction. The terminal is a text-based way to issue all kinds of commands to your computer (not just Python commands) and knowing a little bit about it is really useful for coding (and more) because managing packages, environments (which we haven't yet discussed), and version control (ditto) can all be done via the terminal. We'll come to these in due course in the chapter on @sec-command-line, but for now, a little background on what the terminal is and what it does.\n", - "\n", - "::: {.callout-note}\n", - "To open up the command line within Visual Studio Code, use the ⌃ + \\` keyboard shortcut (Mac) or ctrl + \\` (Windows/Linux), or click \"View > Terminal\".\n", - "\n", - "If you want to open up the command line independently of Visual Studio Code, search for \"Terminal\" on Mac and Linux, and \"Powershell\" on Windows.\n", - ":::\n", - "\n", - "Firstly, everything you can do by clicking on icons to launch programmes on your computer, you can also do via the terminal, also known as the command line. For many programmes, a lot of their functionality can be accessed using the command line, and other programmes *only* have a command line interface (CLI), including some that are used for data science.\n", - "\n", - "::: {.callout-tip}\n", - "The command line interacts with your operating system and is used to create, activate, or change Python installations.\n", - ":::\n", - "\n", - "Use Visual Studio Code to open a terminal window by clicking Terminal -> New Terminal on the list of commands at the very top of the window. If you have installed uv on your computer, your terminal should look something like this as your 'command prompt':\n", - "\n", - "```bash\n", - "your-username@your-computer current-directory %\n", - "```\n", - "\n", - "on Mac, and the same but with '%' replaced by '$' on linux, and (using Powershell)\n", - "\n", - "```powershell\n", - "PS C:\\Windows\\System32>\n", - "```\n", - "\n", - "on Windows.\n", - "\n", - "You can check that uv has successfully installed Python in your current project's folder by running\n", - "\n", - "```bash\n", - "uv run python --version\n", - "```\n", - "\n", - "For now, to at least try out the command line, let's use something that works across all three of the major operating systems. Type `uv run python` on the command prompt that came up in your new terminal window. You should see information about your installation of Python appear, including the version, followed by a Python prompt that looks like `>>>`. This is a kind of interactive Python session, in the terminal. It's much less rich than the one available in Visual Studio Code (it can't run scripts line-by-line, for example) but you can try `print('Hello World!')` and it will run, printing your message. To exit the terminal-based Python session, type `exit()` to go back to the regular command line.\n", - "\n", - "### Installing Packages\n", - "\n", - "To install extra Python packages, the default and easiest way is to use `uv add **packagename**`. There are over 330,000 Python packages on PyPI (the Python Package Index)! You can see what packages you have installed already by running `uv pip list` into the command line.\n", - "\n", - "`uv add ...` will install packages into the special Python environment in your current folder (it sits in a subdirectory called \".venv\" which will be hidden by default on most systems.) It's really helpful and good practice to have one Python environment per project, and **uv** does this automatically for you.\n", - "\n", - "::: {.callout-tip title=\"Exercise\"}\n", - "Try installing the **matplotlib**, **pandas**, **statsmodels**, and **skimpy** packages using `uv add`.\n", - ":::\n", - "\n", - "### Using Packages\n", - "\n", - "Once you have installed a package, you need to be able to use it! This is usually done via an import statement at the top of your script or Jupyter Notebook. For example, to bring in **pandas**, it's\n", - "\n", - "```python\n", - "import pandas as pd\n", - "```\n", - "\n", - "Why does Python do this? The idea of not just loading every package is to provide clarity over what function is being called from what package. It's also not necessary to load every package for every piece of analysis, and you often actually want to know what the *minimum* set of packages is to reproduce an analysis. Making the package imports explicit helps with all of that.\n", - "\n", - "You may also wonder why one doesn't just use `import pandas as pandas`. There's actually nothing stopping you doing this except i) it's convenient to have a shorter name and ii) there does tend to be a convention around imports, ie `pd` for **pandas** and `np` for **numpy**, and your code will be clearer to yourself and others if you follow the conventions.\n", - "\n", - "## Virtual Code Environments\n", - "\n", - "Virtual code environments allow you to isolate all of the packages that you're using to do analysis for one project from the set of packages you might need for a different project. They're an important part of creating a reproducible analytical pipeline but a key benefit is that others can reproduce the environment you used and it's best practice to have an isolated environment per project.\n", - "\n", - "To be more concrete, let's say you're using Python 3.9, **statsmodels**, and **pandas** for one project, project A. And, for project B, you need to use Python 3.10 with **numpy** and **scikit-learn**. Even with the same version of Python, best practice would be to have two separate virtual Python environments: environment A, with everything needed for project A, and environment B, with everything needed for project B. For the case where you're using different versions of Python, this isn't just best practice, it's essential.\n", - "\n", - "Many programming languages now come with an option to install packages and a version of the language in isolated environments. In Python, there are multiple tools for managing different environments. And, of those, the easiest to work with is probably [**uv**](https://docs.astral.sh/uv/).\n", - "\n", - "You can see all of the packages in the environment created in your current folder by running `uv pip list` on the command line. Here's an example of looking at the installed packages within this very book, filtering them just to the ones beginning with \"s\".\n", - "\n", - "```{bash}\n", - "uv run pip list | grep ^s\n", - "```\n", - "\n", - "### The pyproject.toml file in Python Environments\n", - "\n", - "You may have noticed that a file called `pyproject.toml` has been created." - ], - "id": "8b889898" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "import toml\n", - "from rich import print_json\n", - "\n", - "print_json(data=toml.load(\"pyproject.toml\"))" - ], - "id": "688f09f1", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This lists all of the dependencies, and the version, of a **uv** Python project. There are lots of benefits to tracking what versions of packages you're using like this. One of the most important is that you can *share* projects with other people, and they can install them from these files too.\n", - "\n", - "As you install or remove packages, the `pyproject.toml` file changes in lockstep.\n", - "\n", - "Noe that Visual Studio Code shows which Python environment you are using when you open a Python script or Jupyter Notebook.\n", - "\n", - "![A typical user view in Visual Studio Code](https://github.com/aeturrell/coding-for-economists/blob/main/img/vscode_layout.png?raw=true)\n", - "\n", - "In the screenshot above, you can see the project-environment in two places: on the blue bar at the bottom of the screen, and (in 5), at the top right hand side of the interactive window. A similar top right indicator is present when you have a Jupyter Notebook open too." - ], - "id": "148595b3" - } - ], - "metadata": { - "kernelspec": { - "name": "python3", - "language": "python", - "display_name": "Python 3 (ipykernel)", - "path": "/Users/omagic/Documents/GitHub/python4DSpolars/.venv/share/jupyter/kernels/python3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file From 38d6a836b0bb1a1b66c3113dbade0e0291f45f33 Mon Sep 17 00:00:00 2001 From: Uchenna Ugoh Date: Thu, 23 Apr 2026 23:43:00 +0100 Subject: [PATCH 24/34] Refactor code structure for improved readability and maintainability --- rectangling.ipynb | 70 ++++---- spreadsheets.ipynb | 416 +++------------------------------------------ 2 files changed, 62 insertions(+), 424 deletions(-) diff --git a/rectangling.ipynb b/rectangling.ipynb index 199c215..e565c1d 100644 --- a/rectangling.ipynb +++ b/rectangling.ipynb @@ -41,7 +41,7 @@ "source": [ "### Prerequisites\n", "\n", - "This chapter will use the **polars** data analysis package." + "This chapter will use the **polars** data analysis package.\n" ] }, { @@ -51,7 +51,7 @@ "source": [ "## Lists\n", "\n", - "Lists are a really useful way to work with lots of data at once. They're defined with square brackets, with entries separated by commas. " + "Lists are a really useful way to work with lots of data at once. They're defined with square brackets, with entries separated by commas.\n" ] }, { @@ -70,7 +70,7 @@ "id": "29b10d07", "metadata": {}, "source": [ - "You can also construct them by appending entries:" + "You can also construct them by appending entries:\n" ] }, { @@ -89,7 +89,7 @@ "id": "d8d4f6ed", "metadata": {}, "source": [ - "And you can access earlier entries using an index, which begins at 0 and ends at one less than the length of the list (this is the convention in many programming languages). For instance, to print specific entries at the start, using `0`, and end, using `-1`:" + "And you can access earlier entries using an index, which begins at 0 and ends at one less than the length of the list (this is the convention in many programming languages). For instance, to print specific entries at the start, using `0`, and end, using `-1`:\n" ] }, { @@ -110,7 +110,7 @@ "source": [ "::: {.callout-tip title=\"Exercise\"}\n", "How might you access the penultimate entry in a list object if you didn't know how many elements it had?\n", - ":::" + ":::\n" ] }, { @@ -118,7 +118,7 @@ "id": "6aea9157", "metadata": {}, "source": [ - "As well as accessing positions in lists using indexing, you can use *slices* on lists. This uses the colon character, `:`, to stand in for 'from the beginning' or 'until the end' (when only appearing once). For instance, to print just the last two entries, we would use the index `-2:` to mean from the second-to-last onwards. Here are two distinct examples: getting the first three and last three entries to be successively printed:" + "As well as accessing positions in lists using indexing, you can use _slices_ on lists. This uses the colon character, `:`, to stand in for 'from the beginning' or 'until the end' (when only appearing once). For instance, to print just the last two entries, we would use the index `-2:` to mean from the second-to-last onwards. Here are two distinct examples: getting the first three and last three entries to be successively printed:\n" ] }, { @@ -137,7 +137,7 @@ "id": "c82b5c4a", "metadata": {}, "source": [ - "Slicing can be even more elaborate than that because we can jump entries using a second colon. Here's a full example that begins at the second entry (remember the index starts at 0), runs up until the second-to-last entry (exclusive), and jumps every other entry inbetween (range just produces a list of integers from the value to one less than the last):" + "Slicing can be even more elaborate than that because we can jump entries using a second colon. Here's a full example that begins at the second entry (remember the index starts at 0), runs up until the second-to-last entry (exclusive), and jumps every other entry inbetween (range just produces a list of integers from the value to one less than the last):\n" ] }, { @@ -159,7 +159,7 @@ "id": "813e09bc", "metadata": {}, "source": [ - "A handy trick is that you can print a reversed list entirely using double colons:" + "A handy trick is that you can print a reversed list entirely using double colons:\n" ] }, { @@ -179,7 +179,7 @@ "source": [ "::: {.callout-tip title=\"Exercise\"}\n", "Slice the `list_example` from earlier to get only the first five entries.\n", - ":::" + ":::\n" ] }, { @@ -187,7 +187,7 @@ "id": "b6ff3ca4", "metadata": {}, "source": [ - "What's amazing about lists is that they can hold any type, including other lists! Here's a valid example of a list that's got a lot going on:" + "What's amazing about lists is that they can hold any type, including other lists! Here's a valid example of a list that's got a lot going on:\n" ] }, { @@ -217,7 +217,7 @@ "source": [ "### Hierarchical Data in Lists\n", "\n", - "Because lists can contain more lists (and so on), they can be used to put hierachical data in. Let's take a look at an example:" + "Because lists can contain more lists (and so on), they can be used to put hierachical data in. Let's take a look at an example:\n" ] }, { @@ -236,7 +236,7 @@ "id": "57a81b53", "metadata": {}, "source": [ - "Now, say we wanted to reduce this to a single list. We can do it with a *list comprehension*:" + "Now, say we wanted to reduce this to a single list. We can do it with a _list comprehension_:\n" ] }, { @@ -254,7 +254,7 @@ "id": "8e96185a", "metadata": {}, "source": [ - "What we're saying here is take all of the values of every little list and put them into a single list." + "What we're saying here is take all of the values of every little list and put them into a single list.\n" ] }, { @@ -264,7 +264,7 @@ "source": [ "### From Lists to Data Frames\n", "\n", - "Occassionally, you'll have data in lists that you wish to turn into a data frame. For example, perhaps you have a list of lists like this:" + "Occassionally, you'll have data in lists that you wish to turn into a data frame. For example, perhaps you have a list of lists like this:\n" ] }, { @@ -282,7 +282,7 @@ "id": "fcfc2d3c", "metadata": {}, "source": [ - "You can pass this straight into a constructor for a data frame as the `data=` keyword argument (adding in other info as necessary). Note that this is four lists of three entries, so the inner loop has entries in 0 to 2... it is this inner loop that will be used as the *rows* of any data frame with the number of entries in each inner list equal to the number of *columns*." + "You can pass this straight into a constructor for a data frame as the `data=` keyword argument (adding in other info as necessary). Note that this is four lists of three entries, so the inner loop has entries in 0 to 2... it is this inner loop that will be used as the _rows_ of any data frame with the number of entries in each inner list equal to the number of _columns_.\n" ] }, { @@ -294,7 +294,7 @@ "source": [ "import polars as pl\n", "\n", - "df = pl.DataFrame(data=list_of_lists, schema=[\"a\", \"b\", \"c\"])" + "df = pl.DataFrame(data=list_of_lists, schema=[\"a\", \"b\", \"c\", \"d\"])" ] }, { @@ -302,7 +302,7 @@ "id": "cc797c89", "metadata": {}, "source": [ - "There's one more trick to show you: explode. This is useful when you have data that has more than one level of list depth. Let's say you read in some data with a complex hierarchical structure like this:" + "There's one more trick to show you: explode. This is useful when you have data that has more than one level of list depth. Let's say you read in some data with a complex hierarchical structure like this:\n" ] }, { @@ -328,7 +328,7 @@ "id": "91bb97aa", "metadata": {}, "source": [ - "We have multiple rows and columns that contain lists. In some situations, it's fine to have a list in a column but here it's probably not as it's mixed in with other types of data. We can use `explode()` to split out the columns further length-wise" + "We have multiple rows and columns that contain lists. In some situations, it's fine to have a list in a column but here it's probably not as it's mixed in with other types of data. We can use `explode()` to split out the columns further length-wise\n" ] }, { @@ -338,7 +338,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.explode(\"alpha\")" + "df.explode(\"gamma\")" ] }, { @@ -353,7 +353,7 @@ "The table below compares the different data types found in Python and JSON.\n", "\n", "| JSON OBJECT | PYTHON OBJECT |\n", - "|---------------|---------------|\n", + "| ------------- | ------------- |\n", "| object | dict |\n", "| array | list |\n", "| string | str |\n", @@ -363,9 +363,9 @@ "| true | True |\n", "| false | False |\n", "\n", - "There are typically two operations you may want to do with JSON data: 1) turn JSON data in a Python object (eg JSON to Python dictionary) or vice versa (known as deserialisation and serialisation respectively); and 2) converting a deserialised object into a *different* kind of Python object.\n", + "There are typically two operations you may want to do with JSON data: 1) turn JSON data in a Python object (eg JSON to Python dictionary) or vice versa (known as deserialisation and serialisation respectively); and 2) converting a deserialised object into a _different_ kind of Python object.\n", "\n", - "Let's look at each in turn." + "Let's look at each in turn.\n" ] }, { @@ -379,7 +379,7 @@ "\n", "#### From the Web\n", "\n", - "We'll get some JSON data from an API. Let's grab the latest UK unemployment data (timeseries code \"MGSX\" and dataset code \"LMS\")." + "We'll get some JSON data from an API. Let's grab the latest UK unemployment data (timeseries code \"MGSX\" and dataset code \"LMS\").\n" ] }, { @@ -402,7 +402,7 @@ "id": "051d3b4a", "metadata": {}, "source": [ - "Let's check what type we got:" + "Let's check what type we got:\n" ] }, { @@ -422,7 +422,7 @@ "source": [ "As expected, the JSON data has automatically been read in as a dictionary—but be wary that the fields have been read in as text rather than numbers, datetimes, and other specific data types.\n", "\n", - "We could print the whole object out but that would take up a lot of space; instead let's look at a couple of entries under the \"months\" key." + "We could print the whole object out but that would take up a lot of space; instead let's look at a couple of entries under the \"months\" key.\n" ] }, { @@ -442,7 +442,7 @@ "source": [ "#### From a File or Stream\n", "\n", - "For this exercise, you'll need to download the JSON file 'cakes.json' from the [data folder of the repository](https://github.com/aeturrell/python4DS/tree/main/data) associated with this book and save it in a sub-folder called \"data\". We can take a peek at the data using the terminal (which is what the preceeding exclamation mark means):" + "For this exercise, you'll need to download the JSON file 'cakes.json' from the [data folder of the repository](https://github.com/aeturrell/python4DS/tree/main/data) associated with this book and save it in a sub-folder called \"data\". We can take a peek at the data using the terminal (which is what the preceeding exclamation mark means):\n" ] }, { @@ -468,7 +468,7 @@ "id": "0c664ab6", "metadata": {}, "source": [ - "We use the built-in **json** library to read this into Python (you could also use a file path here—more on how in a moment):" + "We use the built-in **json** library to read this into Python (you could also use a file path here—more on how in a moment):\n" ] }, { @@ -489,7 +489,7 @@ "id": "df41f92b", "metadata": {}, "source": [ - "Note that not everything is the same in going from JSON text to a Python dictionary: JSON uses `null` rather than `None`, won't accept trailing commas at the end of lists, and has basic types that are lists, strings (and all keys must be strings), numbers, booleans, and nulls. Let's now see how to write a Python dictionary back to a JSON, perhaps for writing to file:" + "Note that not everything is the same in going from JSON text to a Python dictionary: JSON uses `null` rather than `None`, won't accept trailing commas at the end of lists, and has basic types that are lists, strings (and all keys must be strings), numbers, booleans, and nulls. Let's now see how to write a Python dictionary back to a JSON, perhaps for writing to file:\n" ] }, { @@ -508,7 +508,7 @@ "id": "5f9445b8", "metadata": {}, "source": [ - "To write to a file, you would use the pattern:" + "To write to a file, you would use the pattern:\n" ] }, { @@ -519,7 +519,7 @@ "```python\n", "with open('data/json_data_output.json', 'w') as outfile:\n", " json.dump(json_stream, outfile)\n", - "```" + "```\n" ] }, { @@ -531,7 +531,7 @@ "\n", "```python\n", "json.load(open(\"data/json_data_output.json\"))\n", - "```" + "```\n" ] }, { @@ -561,7 +561,7 @@ "id": "a1346020", "metadata": {}, "source": [ - "The web-scraped data we downloaded earlier had a more complicated structure, but **polars** has a `json_normalize()` function that can cope with this. For example, with the following data, there are many missing entries but `json_normalize()` can still parse it into a Data Frame." + "The web-scraped data we downloaded earlier had a more complicated structure, but **polars** has a `json_normalize()` function that can cope with this. For example, with the following data, there are many missing entries but `json_normalize()` can still parse it into a Data Frame.\n" ] }, { @@ -584,7 +584,7 @@ "id": "7eaf00e1", "metadata": {}, "source": [ - "And we can control the level that properties like 'name' are split out to as well (you can check out more options over at the [**polars** documentation](https://docs.pola.rs/api/python/stable/reference/api/polars.json_normalize.html))" + "And we can control the level that properties like 'name' are split out to as well (you can check out more options over at the [**polars** documentation](https://docs.pola.rs/api/python/stable/reference/api/polars.json_normalize.html))\n" ] }, { @@ -602,7 +602,7 @@ "id": "78d637e5", "metadata": {}, "source": [ - "As well as the JSON normalise function, **polars** has a `from_dict()` method to work with simpler dictionary objects." + "As well as the JSON normalise function, **polars** has a `from_dict()` method to work with simpler dictionary objects.\n" ] } ], @@ -614,7 +614,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "python4ds", "language": "python", "name": "python3" }, diff --git a/spreadsheets.ipynb b/spreadsheets.ipynb index ee7f5bf..1c1f378 100644 --- a/spreadsheets.ipynb +++ b/spreadsheets.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "51a55374", "metadata": { "tags": [ @@ -68,43 +68,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "bf00d4c6", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 5)
Student IDFull Namefavourite.foodmealPlanAGE
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only"null
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""five"
6"Güvenç Attila""Ice cream""Lunch only""6"
" - ], - "text/plain": [ - "shape: (6, 5)\n", - "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", - "│ Student ID ┆ Full Name ┆ favourite.food ┆ mealPlan ┆ AGE │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", - "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", - "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", - "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", - "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", - "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ null │\n", - "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ five │\n", - "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", - "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import polars as pl\n", "\n", @@ -126,43 +93,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "ecce26ea", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 5)
student_idfull_namefavourite_foodmeal_planage
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only"null
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""five"
6"Güvenç Attila""Ice cream""Lunch only""6"
" - ], - "text/plain": [ - "shape: (6, 5)\n", - "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", - "│ student_id ┆ full_name ┆ favourite_food ┆ meal_plan ┆ age │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", - "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", - "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", - "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", - "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", - "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ null │\n", - "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ five │\n", - "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", - "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "students.columns = [\n", " \"student_id\",\n", @@ -184,43 +118,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "22e45cc2", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 5)
student_idfull_namefavourite_foodmeal_planage
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only"null
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""5"
6"Güvenç Attila""Ice cream""Lunch only""6"
" - ], - "text/plain": [ - "shape: (6, 5)\n", - "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", - "│ student_id ┆ full_name ┆ favourite_food ┆ meal_plan ┆ age │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", - "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", - "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", - "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", - "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", - "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ null │\n", - "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ 5 │\n", - "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", - "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "students = pl.read_excel(\"data/students.xlsx\")\n", "students.columns = [\n", @@ -244,25 +145,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "f67490d3", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Schema([('student_id', Int64),\n", - " ('full_name', String),\n", - " ('favourite_food', String),\n", - " ('meal_plan', Categorical(ordering='physical')),\n", - " ('age', Int64)])" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "students = students.with_columns(\n", " [\n", @@ -300,43 +186,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "e37f9e3d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (5, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64i64i64stri64
"Adelie""Torgersen"39.118.71813750"male"2007
"Adelie""Torgersen"39.517.41863800"female"2007
"Adelie""Torgersen"40.318.01953250"female"2007
"Adelie""Torgersen"nullnullnullnull"NA"2007
"Adelie""Torgersen"36.719.31933450"female"2007
" - ], - "text/plain": [ - "shape: (5, 8)\n", - "┌─────────┬───────────┬───────────────┬───────────────┬──────────────┬─────────────┬────────┬──────┐\n", - "│ species ┆ island ┆ bill_length_m ┆ bill_depth_mm ┆ flipper_leng ┆ body_mass_g ┆ sex ┆ year │\n", - "│ --- ┆ --- ┆ m ┆ --- ┆ th_mm ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ --- ┆ f64 ┆ --- ┆ i64 ┆ str ┆ i64 │\n", - "│ ┆ ┆ f64 ┆ ┆ i64 ┆ ┆ ┆ │\n", - "╞═════════╪═══════════╪═══════════════╪═══════════════╪══════════════╪═════════════╪════════╪══════╡\n", - "│ Adelie ┆ Torgersen ┆ 39.1 ┆ 18.7 ┆ 181 ┆ 3750 ┆ male ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 39.5 ┆ 17.4 ┆ 186 ┆ 3800 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 40.3 ┆ 18.0 ┆ 195 ┆ 3250 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ null ┆ null ┆ null ┆ null ┆ NA ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 36.7 ┆ 19.3 ┆ 193 ┆ 3450 ┆ female ┆ 2007 │\n", - "└─────────┴───────────┴───────────────┴───────────────┴──────────────┴─────────────┴────────┴──────┘" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pl.read_excel(\n", " \"data/penguins.xlsx\",\n", @@ -356,18 +209,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "0f3a3b78", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['Torgersen Island', 'Biscoe Island', 'Dream Island']\n" - ] - } - ], + "outputs": [], "source": [ "penguins_dict = pl.read_excel(\n", " \"data/penguins.xlsx\",\n", @@ -386,50 +231,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "15495426", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Biscoe Island\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "shape: (5, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64i64i64stri64
"Adelie""Biscoe"37.818.31743400"female"2007
"Adelie""Biscoe"37.718.71803600"male"2007
"Adelie""Biscoe"35.919.21893800"female"2007
"Adelie""Biscoe"38.218.11853950"male"2007
"Adelie""Biscoe"38.817.21803800"male"2007
" - ], - "text/plain": [ - "shape: (5, 8)\n", - "┌─────────┬────────┬────────────────┬───────────────┬────────────────┬─────────────┬────────┬──────┐\n", - "│ species ┆ island ┆ bill_length_mm ┆ bill_depth_mm ┆ flipper_length ┆ body_mass_g ┆ sex ┆ year │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ _mm ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ f64 ┆ f64 ┆ --- ┆ i64 ┆ str ┆ i64 │\n", - "│ ┆ ┆ ┆ ┆ i64 ┆ ┆ ┆ │\n", - "╞═════════╪════════╪════════════════╪═══════════════╪════════════════╪═════════════╪════════╪══════╡\n", - "│ Adelie ┆ Biscoe ┆ 37.8 ┆ 18.3 ┆ 174 ┆ 3400 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Biscoe ┆ 37.7 ┆ 18.7 ┆ 180 ┆ 3600 ┆ male ┆ 2007 │\n", - "│ Adelie ┆ Biscoe ┆ 35.9 ┆ 19.2 ┆ 189 ┆ 3800 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Biscoe ┆ 38.2 ┆ 18.1 ┆ 185 ┆ 3950 ┆ male ┆ 2007 │\n", - "│ Adelie ┆ Biscoe ┆ 38.8 ┆ 17.2 ┆ 180 ┆ 3800 ┆ male ┆ 2007 │\n", - "└─────────┴────────┴────────────────┴───────────────┴────────────────┴─────────────┴────────┴──────┘" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "print(list(penguins_dict.keys())[1])\n", "list(penguins_dict.values())[1].head()" @@ -445,49 +250,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "151ba846", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (344, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64i64i64stri64
"Adelie""Torgersen"39.118.71813750"male"2007
"Adelie""Torgersen"39.517.41863800"female"2007
"Adelie""Torgersen"40.318.01953250"female"2007
"Adelie""Torgersen"nullnullnullnull"NA"2007
"Adelie""Torgersen"36.719.31933450"female"2007
……………………
"Chinstrap""Dream"55.819.82074000"male"2009
"Chinstrap""Dream"43.518.12023400"female"2009
"Chinstrap""Dream"49.618.21933775"male"2009
"Chinstrap""Dream"50.819.02104100"male"2009
"Chinstrap""Dream"50.218.71983775"female"2009
" - ], - "text/plain": [ - "shape: (344, 8)\n", - "┌───────────┬───────────┬──────────────┬──────────────┬──────────────┬─────────────┬────────┬──────┐\n", - "│ species ┆ island ┆ bill_length_ ┆ bill_depth_m ┆ flipper_leng ┆ body_mass_g ┆ sex ┆ year │\n", - "│ --- ┆ --- ┆ mm ┆ m ┆ th_mm ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ --- ┆ --- ┆ --- ┆ i64 ┆ str ┆ i64 │\n", - "│ ┆ ┆ f64 ┆ f64 ┆ i64 ┆ ┆ ┆ │\n", - "╞═══════════╪═══════════╪══════════════╪══════════════╪══════════════╪═════════════╪════════╪══════╡\n", - "│ Adelie ┆ Torgersen ┆ 39.1 ┆ 18.7 ┆ 181 ┆ 3750 ┆ male ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 39.5 ┆ 17.4 ┆ 186 ┆ 3800 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 40.3 ┆ 18.0 ┆ 195 ┆ 3250 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ null ┆ null ┆ null ┆ null ┆ NA ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 36.7 ┆ 19.3 ┆ 193 ┆ 3450 ┆ female ┆ 2007 │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ Chinstrap ┆ Dream ┆ 55.8 ┆ 19.8 ┆ 207 ┆ 4000 ┆ male ┆ 2009 │\n", - "│ Chinstrap ┆ Dream ┆ 43.5 ┆ 18.1 ┆ 202 ┆ 3400 ┆ female ┆ 2009 │\n", - "│ Chinstrap ┆ Dream ┆ 49.6 ┆ 18.2 ┆ 193 ┆ 3775 ┆ male ┆ 2009 │\n", - "│ Chinstrap ┆ Dream ┆ 50.8 ┆ 19.0 ┆ 210 ┆ 4100 ┆ male ┆ 2009 │\n", - "│ Chinstrap ┆ Dream ┆ 50.2 ┆ 18.7 ┆ 198 ┆ 3775 ┆ female ┆ 2009 │\n", - "└───────────┴───────────┴──────────────┴──────────────┴──────────────┴─────────────┴────────┴──────┘" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "penguins = pl.concat(penguins_dict.values())\n", "penguins" @@ -519,49 +285,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "29987b9d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (14, 6)
Lots of people__UNNAMED__1__UNNAMED__2__UNNAMED__3__UNNAMED__4__UNNAMED__5
strstrstrstrdatestr
"David Bowie""musician""69""true"1947-01-08"2016-01-10 00:00:00"
"Carrie Fisher""actor""60""true"1956-10-21"2016-12-27 00:00:00"
"Chuck Berry""musician""90""true"1926-10-18"2017-03-18 00:00:00"
"Bill Paxton""actor""61""true"1955-05-17"2017-02-25 00:00:00"
"Prince""musician""57""true"1958-06-07"2016-04-21 00:00:00"
………………
"George Michael""musician""53""false"1963-06-25"2016-12-25 00:00:00"
"Some"nullnullnullnullnull
null"also like to write stuff"nullnullnullnull
nullnull"at the""bottom,"nullnull
nullnullnullnullnull"too!"
" - ], - "text/plain": [ - "shape: (14, 6)\n", - "┌────────────────┬──────────────┬──────────────┬──────────────┬──────────────┬─────────────────────┐\n", - "│ Lots of people ┆ __UNNAMED__1 ┆ __UNNAMED__2 ┆ __UNNAMED__3 ┆ __UNNAMED__4 ┆ __UNNAMED__5 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ str ┆ str ┆ date ┆ str │\n", - "╞════════════════╪══════════════╪══════════════╪══════════════╪══════════════╪═════════════════════╡\n", - "│ David Bowie ┆ musician ┆ 69 ┆ true ┆ 1947-01-08 ┆ 2016-01-10 00:00:00 │\n", - "│ Carrie Fisher ┆ actor ┆ 60 ┆ true ┆ 1956-10-21 ┆ 2016-12-27 00:00:00 │\n", - "│ Chuck Berry ┆ musician ┆ 90 ┆ true ┆ 1926-10-18 ┆ 2017-03-18 00:00:00 │\n", - "│ Bill Paxton ┆ actor ┆ 61 ┆ true ┆ 1955-05-17 ┆ 2017-02-25 00:00:00 │\n", - "│ Prince ┆ musician ┆ 57 ┆ true ┆ 1958-06-07 ┆ 2016-04-21 00:00:00 │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ George Michael ┆ musician ┆ 53 ┆ false ┆ 1963-06-25 ┆ 2016-12-25 00:00:00 │\n", - "│ Some ┆ null ┆ null ┆ null ┆ null ┆ null │\n", - "│ null ┆ also like to ┆ null ┆ null ┆ null ┆ null │\n", - "│ ┆ write stuff ┆ ┆ ┆ ┆ │\n", - "│ null ┆ null ┆ at the ┆ bottom, ┆ null ┆ null │\n", - "│ null ┆ null ┆ null ┆ null ┆ null ┆ too! │\n", - "└────────────────┴──────────────┴──────────────┴──────────────┴──────────────┴─────────────────────┘" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pl.read_excel(\n", " \"data/deaths.xlsx\",\n", @@ -579,47 +306,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "9d7a3db3", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (10, 6)
Lots of people__UNNAMED__1__UNNAMED__2__UNNAMED__3__UNNAMED__4__UNNAMED__5
strstri64booldatedate
"David Bowie""musician"69true1947-01-082016-01-10
"Carrie Fisher""actor"60true1956-10-212016-12-27
"Chuck Berry""musician"90true1926-10-182017-03-18
"Bill Paxton""actor"61true1955-05-172017-02-25
"Prince""musician"57true1958-06-072016-04-21
"Alan Rickman""actor"69false1946-02-212016-01-14
"Florence Henderson""actor"82true1934-02-142016-11-24
"Harper Lee""author"89false1926-04-282016-02-19
"Zsa Zsa Gábor""actor"99true1917-02-062016-12-18
"George Michael""musician"53false1963-06-252016-12-25
" - ], - "text/plain": [ - "shape: (10, 6)\n", - "┌────────────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────────┐\n", - "│ Lots of people ┆ __UNNAMED__1 ┆ __UNNAMED__2 ┆ __UNNAMED__3 ┆ __UNNAMED__4 ┆ __UNNAMED__5 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ i64 ┆ bool ┆ date ┆ date │\n", - "╞════════════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════════╡\n", - "│ David Bowie ┆ musician ┆ 69 ┆ true ┆ 1947-01-08 ┆ 2016-01-10 │\n", - "│ Carrie Fisher ┆ actor ┆ 60 ┆ true ┆ 1956-10-21 ┆ 2016-12-27 │\n", - "│ Chuck Berry ┆ musician ┆ 90 ┆ true ┆ 1926-10-18 ┆ 2017-03-18 │\n", - "│ Bill Paxton ┆ actor ┆ 61 ┆ true ┆ 1955-05-17 ┆ 2017-02-25 │\n", - "│ Prince ┆ musician ┆ 57 ┆ true ┆ 1958-06-07 ┆ 2016-04-21 │\n", - "│ Alan Rickman ┆ actor ┆ 69 ┆ false ┆ 1946-02-21 ┆ 2016-01-14 │\n", - "│ Florence Henderson ┆ actor ┆ 82 ┆ true ┆ 1934-02-14 ┆ 2016-11-24 │\n", - "│ Harper Lee ┆ author ┆ 89 ┆ false ┆ 1926-04-28 ┆ 2016-02-19 │\n", - "│ Zsa Zsa Gábor ┆ actor ┆ 99 ┆ true ┆ 1917-02-06 ┆ 2016-12-18 │\n", - "│ George Michael ┆ musician ┆ 53 ┆ false ┆ 1963-06-25 ┆ 2016-12-25 │\n", - "└────────────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┘" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pl.read_excel(\n", " \"data/deaths.xlsx\",\n", @@ -666,40 +356,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "15963e18", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (3, 2)
itemquantity
cati64
"brownie"10
"cupcake"5
"cookie"8
" - ], - "text/plain": [ - "shape: (3, 2)\n", - "┌─────────┬──────────┐\n", - "│ item ┆ quantity │\n", - "│ --- ┆ --- │\n", - "│ cat ┆ i64 │\n", - "╞═════════╪══════════╡\n", - "│ brownie ┆ 10 │\n", - "│ cupcake ┆ 5 │\n", - "│ cookie ┆ 8 │\n", - "└─────────┴──────────┘" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "bake_sale = pl.DataFrame(\n", " {\n", @@ -720,21 +380,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "1fc17141", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "bake_sale.write_excel(\"data/bake_sale.xlsx\")" ] @@ -759,21 +408,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "27e128f9", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Schema([('item', String), ('quantity', Int64)])" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pl.read_excel(\"data/bake_sale.xlsx\").schema" ] From 07dcbb03bd5f3f7e86d82de15d0fb2f5c994e731 Mon Sep 17 00:00:00 2001 From: Uchenna Ugoh Date: Fri, 24 Apr 2026 00:00:35 +0100 Subject: [PATCH 25/34] Refactor notebook content for improved readability and formatting - Added newline characters to the end of several text blocks in `webscraping-and-apis.ipynb` to ensure proper formatting. - Adjusted the structure of code snippets for better clarity and consistency. - Removed unnecessary lines and streamlined code for better presentation. --- rectangling.ipynb | 3 +- webscraping-and-apis.ipynb | 92 +++++++++++++++++++------------------- 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/rectangling.ipynb b/rectangling.ipynb index 8a13f79..e565c1d 100644 --- a/rectangling.ipynb +++ b/rectangling.ipynb @@ -576,8 +576,7 @@ " {\"name\": {\"given\": \"Mark\", \"family\": \"Regner\"}},\n", " {\"id\": 2, \"name\": \"Faye Raker\"},\n", "]\n", - "pl.json_normalize(data)\n", - "\n" + "pl.json_normalize(data)" ] }, { diff --git a/webscraping-and-apis.ipynb b/webscraping-and-apis.ipynb index 0c12ffe..5171a4d 100644 --- a/webscraping-and-apis.ipynb +++ b/webscraping-and-apis.ipynb @@ -10,7 +10,7 @@ "\n", "## Introduction\n", "\n", - "This chapter will show you how to work with online data that is either obtained from webpages via webscraping or more directly over the internet via an API. An important principle is always to use an API if one is available as this is designed to pass information directly into your Python session and will save you a lot of effort." + "This chapter will show you how to work with online data that is either obtained from webpages via webscraping or more directly over the internet via an API. An important principle is always to use an API if one is available as this is designed to pass information directly into your Python session and will save you a lot of effort.\n" ] }, { @@ -56,7 +56,7 @@ "\n", "As a brief example, in the US, lists of ingredients and instructions are not copyrightable, so copyright can not be used to protect a recipe. But if that list of recipes is accompanied by substantial novel literary content, that is copyrightable. This is why when you’re looking for a recipe on the internet there’s always so much content beforehand.\n", "\n", - "If you do need to scrape original content (like text or images), you may still be protected under the doctrine of fair use. Fair use is not a hard and fast rule, but weighs up a number of factors. It’s more likely to apply if you are collecting the data for research or non-commercial purposes and if you limit what you scrape to just what you need." + "If you do need to scrape original content (like text or images), you may still be protected under the doctrine of fair use. Fair use is not a hard and fast rule, but weighs up a number of factors. It’s more likely to apply if you are collecting the data for research or non-commercial purposes and if you limit what you scrape to just what you need.\n" ] }, { @@ -69,7 +69,7 @@ "\n", "You will need to install the **pandas** and **polars** package for this chapter. We'll use **seaborn** too, which you should already have installed. You will also need to install the **beautifulsoup**, **pandas-datareader**, and **wbgapi** packages in your terminal using `uv add beautifulsoup4`, and `uv add wbgapi` respectively. We'll also use two built-in packages, **textwrap** and **requests**.\n", "\n", - "To kick off, let's import some of the packages we need (it's always good practice to import the packages you need at the top of a script or notebook)." + "To kick off, let's import some of the packages we need (it's always good practice to import the packages you need at the top of a script or notebook).\n" ] }, { @@ -85,8 +85,7 @@ "import pandas as pd\n", "import polars as pl\n", "import requests\n", - "from bs4 import BeautifulSoup\n", - "import lets_plot as lp" + "from bs4 import BeautifulSoup" ] }, { @@ -97,7 +96,7 @@ "source": [ "## Extracting Data from Files on the Internet using **polars**\n", "\n", - "It's easy to read data from the internet once you have the url and file type. Here, for instance, is an example that reads in the 'storms' dataset, which is stored as a CSV file in a URL (we'll only grab the first 10 rows):" + "It's easy to read data from the internet once you have the url and file type. Here, for instance, is an example that reads in the 'storms' dataset, which is stored as a CSV file in a URL (we'll only grab the first 10 rows):\n" ] }, { @@ -122,7 +121,7 @@ "\n", "Using an API (application programming interface) is another way to draw down information from the interweb. Their just a way for one tool, say Python, to speak to another tool, say a server, and usefully exchange information. The classic use case would be to post a request for data that fits a certain query via an API and to get a download of that data back in return. (You should always preferentially use an API over webscraping a site.)\n", "\n", - "Because they are designed to work with any tool, you don't actually need a programming language to interact with an API, it's just a *lot* easier if you do.\n", + "Because they are designed to work with any tool, you don't actually need a programming language to interact with an API, it's just a _lot_ easier if you do.\n", "\n", "::: {.callout-note}\n", "An API key is needed in order to access some APIs. Sometimes all you need to do is register with site, in other cases you may have to pay for access.\n", @@ -132,7 +131,7 @@ "\n", "An API has an 'endpoint', the base url, and then a URL that encodes the question. Let's see an example with the ONS API for which the endpoint is \"https://api.beta.ons.gov.uk/v1/\". The rest of the API has the form 'data?uri=' and then the long ID of both the timeseries (jp9z) and then the dataset (LMS), which is vacancies in the UK services sector.\n", "\n", - "The data that are returned by APIs are typically in JSON format, which looks a lot like a nested Python dictionary and its entries can be accessed in the same way--this is what is happening when getting the series' title in the example below. JSON is not good for analysis, so we'll use **polars** to put the data into shape." + "The data that are returned by APIs are typically in JSON format, which looks a lot like a nested Python dictionary and its entries can be accessed in the same way--this is what is happening when getting the series' title in the example below. JSON is not good for analysis, so we'll use **polars** to put the data into shape.\n" ] }, { @@ -152,26 +151,29 @@ "# Convert dates using string operations\n", "df = (\n", " pl.DataFrame(json_data[\"months\"])\n", - " .with_columns([\n", - " # Add day to make it a valid date string\n", - " (pl.col(\"date\") + \"-01\").str.to_date(format=\"%Y %b-%d\").alias(\"date\"),\n", - " pl.col(\"value\").cast(pl.Float64).alias(\"value\"),\n", - " ])\n", + " .with_columns(\n", + " [\n", + " # Add day to make it a valid date string\n", + " (pl.col(\"date\") + \"-01\").str.to_date(format=\"%Y %b-%d\").alias(\"date\"),\n", + " pl.col(\"value\").cast(pl.Float64).alias(\"value\"),\n", + " ]\n", + " )\n", " .drop_nulls(\"date\")\n", " .sort(\"date\")\n", ")\n", "\n", "\n", - "\n", "# Initialize the library\n", "lp.LetsPlot.setup_html()\n", "\n", "# Create plot using the alias\n", - "chart = lp.ggplot(df, lp.aes(x=\"date\", y=\"value\")) + \\\n", - " lp.geom_line(size=2.0, color=\"steelblue\") + \\\n", - " lp.ggtitle(title) + \\\n", - " lp.ylim(0, df[\"value\"].max() * 1.2) + \\\n", - " lp.theme_classic()\n", + "chart = (\n", + " lp.ggplot(df, lp.aes(x=\"date\", y=\"value\"))\n", + " + lp.geom_line(size=2.0, color=\"steelblue\")\n", + " + lp.ggtitle(title)\n", + " + lp.ylim(0, df[\"value\"].max() * 1.2)\n", + " + lp.theme_classic()\n", + ")\n", "\n", "chart" ] @@ -182,9 +184,9 @@ "id": "670ce0bb", "metadata": {}, "source": [ - "We've talked about *reading* APIs. You can also create your own to serve up data, models, whatever you like! This is an advanced topic and we won't cover it; but if you do need to, the simplest way is to use [Fast API](https://fastapi.tiangolo.com/). You can find some short video tutorials for Fast API [here](https://calmcode.io/fastapi/hello-world.html).\n", + "We've talked about _reading_ APIs. You can also create your own to serve up data, models, whatever you like! This is an advanced topic and we won't cover it; but if you do need to, the simplest way is to use [Fast API](https://fastapi.tiangolo.com/). You can find some short video tutorials for Fast API [here](https://calmcode.io/fastapi/hello-world.html).\n", "\n", - "### Accessing World Bank Data with wbgapi" + "### Accessing World Bank Data with wbgapi\n" ] }, { @@ -195,7 +197,7 @@ "source": [ "While APIs can be accessed directly using tools like requests, some specialized libraries make working with structured datasets much easier. One such example is wbgapi, which provides a convenient interface for accessing World Bank data.\n", "\n", - "Let’s look at an example using World Bank data on CO₂-equivalent emissions per capita:" + "Let’s look at an example using World Bank data on CO₂-equivalent emissions per capita:\n" ] }, { @@ -224,9 +226,7 @@ " ).reset_index()\n", " )\n", " .rename({\"Country\": \"country\"})\n", - " .with_columns(\n", - " pl.col(\"country\").map_elements(lambda x: textwrap.fill(x, 10))\n", - " )\n", + " .with_columns(pl.col(\"country\").map_elements(lambda x: textwrap.fill(x, 10)))\n", " .sort(indicator_code, descending=True)\n", ")\n", "\n", @@ -280,7 +280,7 @@ "\n", "Positional Keys: Eurostat's REST API expects a \"key string\" where codes are placed in a specific order separated by dots (e.g., Freq.Unit.Item.Geo). If you know the order, you can \"slice\" the data cube directly.\n", "\n", - "Let’s see an example of this in action. We want to see the Harmonised Index of Consumer Prices (HICP)—specifically the annual rate of change for all items—for Germany and France. We will use the resource prc_hicp_manr, requesting Monthly frequency (M), the Annual Rate of Change unit (RCH_A), and the \"All-items\" classification (CP00)." + "Let’s see an example of this in action. We want to see the Harmonised Index of Consumer Prices (HICP)—specifically the annual rate of change for all items—for Germany and France. We will use the resource prc_hicp_manr, requesting Monthly frequency (M), the Annual Rate of Change unit (RCH_A), and the \"All-items\" classification (CP00).\n" ] }, { @@ -316,7 +316,7 @@ "df = pl.from_pandas(df_pd)\n", "\n", "print(df.head())\n", - "```" + "```\n" ] }, { @@ -325,13 +325,13 @@ "id": "e5cac233", "metadata": {}, "source": [ - "| | TIME_PERIOD | geo | unit | freq | coicop | value |\n", - "|--:|------------:|:----|:------|:-----|:-------|------:|\n", - "| 0 | 2024-01 | DE | RCH_A | M | CP00 | 3.1 |\n", - "| 1 | 2024-02 | DE | RCH_A | M | CP00 | 2.7 |\n", - "| 2 | 2024-03 | DE | RCH_A | M | CP00 | 2.3 |\n", - "| 3 | 2024-04 | DE | RCH_A | M | CP00 | 2.4 |\n", - "| 4 | 2024-05 | DE | RCH_A | M | CP00 | 2.8 |" + "| | TIME_PERIOD | geo | unit | freq | coicop | value |\n", + "| --: | ----------: | :-- | :---- | :--- | :----- | ----: |\n", + "| 0 | 2024-01 | DE | RCH_A | M | CP00 | 3.1 |\n", + "| 1 | 2024-02 | DE | RCH_A | M | CP00 | 2.7 |\n", + "| 2 | 2024-03 | DE | RCH_A | M | CP00 | 2.3 |\n", + "| 3 | 2024-04 | DE | RCH_A | M | CP00 | 2.4 |\n", + "| 4 | 2024-05 | DE | RCH_A | M | CP00 | 2.8 |\n" ] }, { @@ -340,7 +340,7 @@ "id": "302326b4", "metadata": {}, "source": [ - "Great that worked! We have data in a nice tidy format." + "Great that worked! We have data in a nice tidy format.\n" ] }, { @@ -354,7 +354,7 @@ "- There is a regularly updated list of APIs over at this [public APIs repo on github](https://github.com/public-apis/public-apis). It doesn't have an economics section (yet), but it has a LOT of other APIs.\n", "- Berkeley Library maintains a [list of economics APIs](https://guides.lib.berkeley.edu/c.php?g=4395&p=7995952) that is well worth looking through.\n", "- [NASDAQ Data Link](https://docs.data.nasdaq.com/), which has a great deal of [financial data](https://docs.data.nasdaq.com/docs/data-organization).\n", - "- [DBnomics](https://db.nomics.world/): publicly-available economic data provided by national and international statistical institutions, but also by researchers and private companies." + "- [DBnomics](https://db.nomics.world/): publicly-available economic data provided by national and international statistical institutions, but also by researchers and private companies.\n" ] }, { @@ -367,7 +367,7 @@ "\n", "Webscraping is a way of grabbing information from the internet that was intended to be displayed in a browser. But it should only be used as a last resort, and only then when permitted by the terms and conditions of a website.\n", "\n", - "If you're getting data from the internet, it's much better to use an API whenever you can: grabbing information in a structure way is *exactly* why APIs exist. APIs should also be more stable than websites, which may change frequently. Typically, if an organisation is happy for you to grab their data, they will have made an API expressly for that purpose. It's pretty rare that there's a major website which *does* permit webscraping but which doesn't have an API; for these websites, if they don't have an API, chances scraping is against their terms and conditions. Those terms and conditions may be enforceable by law (different rules in different countries here, and you really need legal advice if it's not unambiguous as to whether you can scrape or not.)\n", + "If you're getting data from the internet, it's much better to use an API whenever you can: grabbing information in a structure way is _exactly_ why APIs exist. APIs should also be more stable than websites, which may change frequently. Typically, if an organisation is happy for you to grab their data, they will have made an API expressly for that purpose. It's pretty rare that there's a major website which _does_ permit webscraping but which doesn't have an API; for these websites, if they don't have an API, chances scraping is against their terms and conditions. Those terms and conditions may be enforceable by law (different rules in different countries here, and you really need legal advice if it's not unambiguous as to whether you can scrape or not.)\n", "\n", "There are other reasons why webscraping is not so good; for example, if you need a back-run then it might be offered through an API but not shown on the webpage. (Or it might not be available at all, in which case it's best to get in touch with the organisation or check out WaybackMachine in case they took snapshots).\n", "\n", @@ -375,13 +375,13 @@ "\n", "If you do find yourself in a scraping situation, be really sure to check that's legally allowed and also that you are not violating the website's `robots.txt` rules: this is a special file on almost every website that sets out what's fair play to crawl (conditional on legality) and what robots should not go poking around in.\n", "\n", - "In Python, you are spoiled for choice when it comes to webscraping. There are five very strong libraries that cover a real range of user styles and needs: **requests**, **lxml**, **beautifulsoup**, **selenium**, and *scrapy**.\n", + "In Python, you are spoiled for choice when it comes to webscraping. There are five very strong libraries that cover a real range of user styles and needs: **requests**, **lxml**, **beautifulsoup**, **selenium**, and \\*scrapy\\*\\*.\n", "\n", "For quick and simple webscraping, my usual combo would **requests**, which does little more than go and grab the HTML of a webpage, and **beautifulsoup**, which then helps you to navigate the structure of the page and pull out what you're actually interested in. For dynamic webpages that use javascript rather than just HTML, you'll need **selenium**. To scale up and hit thousands of webpages in an efficient way, you might try **scrapy**, which can work with the other tools and handle multiple sessions, and all other kinds of bells and whistles... it's actually a \"web scraping framework\".\n", "\n", "It's always helpful to see coding in practice, so that's what we'll do now, but note that we'll be skipping over a lot of important detail such as user agents, being 'polite' with your scraping requests, being efficient with caching and crawling.\n", "\n", - "In lieu of a better example, let's scrape the research page of [http://aeturrell.com/](http://aeturrell.com/)" + "In lieu of a better example, let's scrape the research page of [http://aeturrell.com/](http://aeturrell.com/)\n" ] }, { @@ -404,7 +404,7 @@ "source": [ "Okay, what just happened? We asked requests to grab the HTML of the webpage and then printed the first 300 characters of the text that it found.\n", "\n", - "Let's now parse this into something humans can read (or can read more easily) using beautifulsoup:" + "Let's now parse this into something humans can read (or can read more easily) using beautifulsoup:\n" ] }, { @@ -424,7 +424,7 @@ "id": "5748e928", "metadata": {}, "source": [ - "Now we see more structure of the page and even some *HTML tags* such as 'title' and 'link'. Now we come to the data extraction part: say we want to pull out every paragraph of text, we can use beautifulsoup to skim down the HTML structure and pull out only those parts with the paragraph tag ('p').\n" + "Now we see more structure of the page and even some _HTML tags_ such as 'title' and 'link'. Now we come to the data extraction part: say we want to pull out every paragraph of text, we can use beautifulsoup to skim down the HTML structure and pull out only those parts with the paragraph tag ('p').\n" ] }, { @@ -446,7 +446,7 @@ "id": "2936677e", "metadata": {}, "source": [ - "Although this paragraph isn't too bad, you can make this more readable by stripping out HTML tags altogether with the `.text` method:" + "Although this paragraph isn't too bad, you can make this more readable by stripping out HTML tags altogether with the `.text` method:\n" ] }, { @@ -465,7 +465,7 @@ "id": "9d9d890e", "metadata": {}, "source": [ - "Now let's say we didn't care about most of the page, we *only* wanted to get hold of the names of projects. For this we need to identify the tag type of the element we're interested in, in this case 'div', and it's class type, in this case \"project-name\". We do it like this (and show nice text in the process):\n" + "Now let's say we didn't care about most of the page, we _only_ wanted to get hold of the names of projects. For this we need to identify the tag type of the element we're interested in, in this case 'div', and it's class type, in this case \"project-name\". We do it like this (and show nice text in the process):\n" ] }, { @@ -498,7 +498,7 @@ "info_on_pages = [scraper(root_url + str(i)) for i in range(start, stop)]\n", "```\n", "\n", - "That's all we'll cover here but remember we've barely *scraped* the surface of this big, complex topic. If you want to read about an application, it's hard not to recommend the paper on webscraping that has undoubtedly change the world the most, and very likely has affected your own life in numerous ways: [\"The PageRank Citation Ranking: Bringing Order to the Web\"](http://ilpubs.stanford.edu:8090/422/) by Page, Brin, Motwani and Winograd. For a more in-depth example of webscraping, check out realpython's [tutorial](https://realpython.com/python-web-scraping-practical-introduction/)." + "That's all we'll cover here but remember we've barely _scraped_ the surface of this big, complex topic. If you want to read about an application, it's hard not to recommend the paper on webscraping that has undoubtedly change the world the most, and very likely has affected your own life in numerous ways: [\"The PageRank Citation Ranking: Bringing Order to the Web\"](http://ilpubs.stanford.edu:8090/422/) by Page, Brin, Motwani and Winograd. For a more in-depth example of webscraping, check out realpython's [tutorial](https://realpython.com/python-web-scraping-practical-introduction/).\n" ] }, { @@ -513,7 +513,7 @@ "\n", "We will read data from 'https://webscraper.io/test-sites/tables' using `pd.read_html()`. This function scans the webpage and returns a list of all tables it finds as DataFrames. To target a specific table, we use the match= keyword argument with text that uniquely appears in the table we want—in this case, \"First Name\".\n", "\n", - "Once captured, we convert the result to Polars using pl.from_pandas() to take advantage of Polars' superior query performance and expression API." + "Once captured, we convert the result to Polars using pl.from_pandas() to take advantage of Polars' superior query performance and expression API.\n" ] }, { @@ -540,7 +540,7 @@ "source": [ "This gives us the table neatly loaded into a **polars** data frame ready for further use.\n", "\n", - "If you get a '403' error, it means that the website has blocked **pandas** because it can see that you are engaged in web scraping. This is because some people web scrape irresponsibly, or because websites have provided other, preferred ways for you to obtain the data, eg via a download of the whole thing (think Wikipedia) or through an API. (If you really need to, [you can often get around the 403 error](https://stackoverflow.com/questions/43590153/http-error-403-forbidden-when-reading-html) though.)" + "If you get a '403' error, it means that the website has blocked **pandas** because it can see that you are engaged in web scraping. This is because some people web scrape irresponsibly, or because websites have provided other, preferred ways for you to obtain the data, eg via a download of the whole thing (think Wikipedia) or through an API. (If you really need to, [you can often get around the 403 error](https://stackoverflow.com/questions/43590153/http-error-403-forbidden-when-reading-html) though.)\n" ] } ], From cfd741d25ab450b1a0d897e640f6508b741e6854 Mon Sep 17 00:00:00 2001 From: Nwabueze Ugoh Date: Fri, 24 Apr 2026 10:44:49 +0100 Subject: [PATCH 26/34] Update functions.ipynb to replace pandas with polars, enhance output display for functions, and correct code snippets for clarity. Adjusted execution counts and fixed minor documentation errors. --- functions.ipynb | 155 +++++++++++++----- visualise.quarto_ipynb_1 | 6 +- workflow-help.quarto_ipynb_1 | 10 +- ...w-packages-and-environments.quarto_ipynb_1 | 6 +- 4 files changed, 127 insertions(+), 50 deletions(-) diff --git a/functions.ipynb b/functions.ipynb index 7171d21..4a0032f 100644 --- a/functions.ipynb +++ b/functions.ipynb @@ -31,7 +31,7 @@ "\n", "### Prerequisites\n", "\n", - "You will need the **pandas** and **numpy** packages for this chapter." + "You will need the **polars** and **numpy** packages for this chapter." ] }, { @@ -48,10 +48,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "0450ad6c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hello Ada, and welcome!\n" + ] + } + ], "source": [ "def welcome_message(name):\n", " return f\"Hello {name}, and welcome!\"\n", @@ -73,10 +81,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "dd49bee5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "('Hello student', 4.8)\n", + "('Hello Ada', 4.8)\n" + ] + } + ], "source": [ "def score_message(score, name=\"student\"):\n", " \"\"\"This is a doc-string, a string describing a function.\n", @@ -122,10 +139,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "50dfff24", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function score_message in module __main__:\n", + "\n", + "score_message(score, name='student')\n", + " This is a doc-string, a string describing a function.\n", + " Args:\n", + " score (float): Raw score\n", + " name (str): Name of student\n", + " Returns:\n", + " str: A hello message.\n", + " float: A normalised score.\n", + "\n" + ] + } + ], "source": [ "help(score_message)" ] @@ -160,20 +195,61 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "7a744085", + "execution_count": 4, + "id": "43d10e40", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (10, 4)
abcd
f64f64f64f64
0.9140810.873650.4582790.785412
0.0738862.0167310.020991.0
1.00.7365450.3230560.98617
0.3479981.4784291.00.0
0.0284780.00.3285560.807754
0.01.3396160.2558630.763866
0.3223031.4192460.00.686972
0.4370661.2256020.2708620.951588
0.0916591.0990040.2531110.920187
0.376460.8695870.0566590.264292
" + ], + "text/plain": [ + "shape: (10, 4)\n", + "┌──────────┬──────────┬──────────┬──────────┐\n", + "│ a ┆ b ┆ c ┆ d │\n", + "│ --- ┆ --- ┆ --- ┆ --- │\n", + "│ f64 ┆ f64 ┆ f64 ┆ f64 │\n", + "╞══════════╪══════════╪══════════╪══════════╡\n", + "│ 0.914081 ┆ 0.87365 ┆ 0.458279 ┆ 0.785412 │\n", + "│ 0.073886 ┆ 2.016731 ┆ 0.02099 ┆ 1.0 │\n", + "│ 1.0 ┆ 0.736545 ┆ 0.323056 ┆ 0.98617 │\n", + "│ 0.347998 ┆ 1.478429 ┆ 1.0 ┆ 0.0 │\n", + "│ 0.028478 ┆ 0.0 ┆ 0.328556 ┆ 0.807754 │\n", + "│ 0.0 ┆ 1.339616 ┆ 0.255863 ┆ 0.763866 │\n", + "│ 0.322303 ┆ 1.419246 ┆ 0.0 ┆ 0.686972 │\n", + "│ 0.437066 ┆ 1.225602 ┆ 0.270862 ┆ 0.951588 │\n", + "│ 0.091659 ┆ 1.099004 ┆ 0.253111 ┆ 0.920187 │\n", + "│ 0.37646 ┆ 0.869587 ┆ 0.056659 ┆ 0.264292 │\n", + "└──────────┴──────────┴──────────┴──────────┘" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import numpy as np\n", - "import pandas as pd\n", + "import polars as pl\n", + "\n", + "df = pl.DataFrame(np.random.normal(size=(10, 4)), schema=[\"a\", \"b\", \"c\", \"d\"])\n", "\n", - "df = pd.DataFrame(np.random.normal(size=(10, 4)), columns=[\"a\", \"b\", \"c\", \"d\"])\n", + "df = df.with_columns([\n", + " ((pl.col(\"a\") - pl.col(\"a\").min()) / (pl.col(\"a\").max() - pl.col(\"a\").min())).alias(\"a\"),\n", + " ((pl.col(\"b\") - pl.col(\"b\").min()) / (pl.col(\"b\").max() - pl.col(\"a\").min())).alias(\"b\"),\n", + " ((pl.col(\"c\") - pl.col(\"c\").min()) / (pl.col(\"c\").max() - pl.col(\"c\").min())).alias(\"c\"),\n", + " ((pl.col(\"d\") - pl.col(\"d\").min()) / (pl.col(\"d\").max() - pl.col(\"d\").min())).alias(\"d\")\n", + "])\n", "\n", - "df[\"a\"] = (df[\"a\"] - df[\"a\"].min()) / (df[\"a\"].max() - df[\"a\"].min())\n", - "df[\"b\"] = (df[\"b\"] - df[\"b\"].min()) / (df[\"b\"].max() - df[\"a\"].min())\n", - "df[\"c\"] = (df[\"c\"] - df[\"c\"].min()) / (df[\"c\"].max() - df[\"c\"].min())\n", - "df[\"d\"] = (df[\"d\"] - df[\"d\"].min()) / (df[\"d\"].max() - df[\"d\"].min())" + "df" ] }, { @@ -182,19 +258,19 @@ "metadata": {}, "source": [ "You might be able to puzzle out that this rescales each column to have a range from 0 to 1.\n", - "But did you spot the mistake? There was an error when copying-and-pasting the code for `df[\"b\"]`: someone forgot to change an `a` to a `b`.\n", + "But did you spot the mistake? There was an error when copying-and-pasting the code for `\"b\"`: someone forgot to change an `a` to a `b`.\n", "Extracting repeated code out into a function is a good idea because it prevents you from making this type of mistake.\n", "\n", "To write a function you need to first analyse the code.\n", "How many inputs does it have?\n", "\n", "```python\n", - "df[\"a\"] - df[\"a\"].min() / (df[\"a\"].max() - df[\"a\"].min())\n", + "pl.col(\"a\") - pl.col(\"a\").min() / (pl.col(\"a\").max() - pl.col(\"a\").min())\n", "```\n", "\n", - "This code only has one input: `df[\"a\"]`. To make the inputs more clear, it's a good idea to rewrite the code using temporary variables with general names. Here this code only requires a single numeric vector, so we'll call it `x` and put it into a function.\n", + "This code only has one input: `pl.col(\"a\")`. To make the inputs more clear, it’s a good idea to rewrite the code using temporary variables with general names. In Polars, this code requires a column expression, so we’ll call it `x` and put it into a function.\n", "\n", - "Functions begin with a def keyword for ‘define a function’. It then has a name, followed by brackets, (), which may contain function arguments and function keyword arguments. This is followed by a colon. The body of the function is then indented relative to the left-most text. Function arguments are defined in brackets following the name, with different inputs separated by commas. Any outputs are given with the return keyword, again with different variables separated by commas.\n", + "Functions begin with a def keyword for 'define a function'. It then has a name, followed by brackets, (), which contain the function argument (our expression `x`). This is followed by a colon. The body of the function is then indented relative to the left-most text. Any outputs—which in this case will be a transformed expression—are given with the return keyword.\n", "\n", "So, in Python, functions have the form:\n", "\n", @@ -228,7 +304,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "406648b6", "metadata": {}, "outputs": [], @@ -264,21 +340,21 @@ { "cell_type": "code", "execution_count": null, - "id": "b72eafca", + "id": "74834d90", "metadata": {}, "outputs": [], "source": [ - "rescale(pd.Series([-10, 0, 10]))" + "rescale(pl.Series([-10, 0, 10]))" ] }, { "cell_type": "code", "execution_count": null, - "id": "edd018c7", + "id": "8a0e643e", "metadata": {}, "outputs": [], "source": [ - "rescale(pd.Series([1, 2, 3, np.nan, 5]))" + "rescale(pl.Series([1, 2, 3, None, 5]))" ] }, { @@ -294,15 +370,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "ce066e4f", "metadata": {}, "outputs": [], "source": [ - "df[\"a\"] = rescale(df[\"a\"])\n", - "df[\"b\"] = rescale(df[\"b\"])\n", - "df[\"c\"] = rescale(df[\"c\"])\n", - "df[\"d\"] = rescale(df[\"d\"])" + "df = df.with_columns([\n", + " rescale(pl.col(\"a\")).alias(\"a\"),\n", + " rescale(pl.col(\"b\")).alias(\"b\"),\n", + " rescale(pl.col(\"c\")).alias(\"c\"),\n", + " rescale(pl.col(\"d\")).alias(\"d\")\n", + "])" ] }, { @@ -319,11 +397,11 @@ { "cell_type": "code", "execution_count": null, - "id": "f503c36b", + "id": "f8b80894", "metadata": {}, "outputs": [], "source": [ - "rescale(pd.Series([1, 2, 3, np.inf, 5]))" + "rescale(pl.Series([1, 2, 3, float('inf'), 5], strict=False))" ] }, { @@ -337,17 +415,16 @@ { "cell_type": "code", "execution_count": null, - "id": "67c861b6", + "id": "cd63ddea", "metadata": {}, "outputs": [], "source": [ "def rescale(x):\n", - " x = x.replace(np.inf, np.nan)\n", + " x = x.cast(pl.Float64).replace(float('inf'), None)\n", " minimum = x.min()\n", " return (x - minimum) / (x.max() - minimum)\n", "\n", - "\n", - "rescale(pd.Series([1, 2, 3, np.inf, 5]))" + "rescale(pl.Series([1, 2, 3, float('inf'), 5], strict=False))" ] }, { @@ -394,7 +471,7 @@ "text_input()\n", "```\n", "\n", - "A good example of this design is the **pandas** package: if you don't remember exactly which function you need to read in data, you can type `pd.read_` and jog your memory as the autocomplete brings up the options." + "A good example of this design is the **polars** package: if you don't remember exactly which function you need to read in data, you can type `pl.read_` and jog your memory as the autocomplete brings up the options." ] }, { @@ -427,7 +504,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8725b986", + "id": "647da443", "metadata": {}, "outputs": [], "source": [ @@ -453,7 +530,7 @@ { "cell_type": "code", "execution_count": null, - "id": "b0a4fe15", + "id": "c5fb049d", "metadata": {}, "outputs": [], "source": [ diff --git a/visualise.quarto_ipynb_1 b/visualise.quarto_ipynb_1 index 2ef6d2f..e6622ad 100644 --- a/visualise.quarto_ipynb_1 +++ b/visualise.quarto_ipynb_1 @@ -28,7 +28,7 @@ "\n", "The first of the three kinds is *exploratory data visualisation*, and it's the kind that you do when you're looking and data and trying to understand it. Just plotting the data is a really good strategy for getting a feel for any issues there might be. This is perhaps most famously demonstrated by Anscombe's quartet: four different datasets with the same mean, standard deviation, and correlation but very different data distributions." ], - "id": "f3331573" + "id": "fe1547a8" }, { "cell_type": "code", @@ -96,7 +96,7 @@ "plt.suptitle(\"Anscombe's Quartet\")\n", "plt.show()" ], - "id": "64a0e7f6", + "id": "a15d7579", "execution_count": null, "outputs": [] }, @@ -120,7 +120,7 @@ "\n", "You can find more information on the topic of communicating via data visualisations in the @sec-communicate-plots chapter." ], - "id": "30b9ff30" + "id": "30c949c1" } ], "metadata": { diff --git a/workflow-help.quarto_ipynb_1 b/workflow-help.quarto_ipynb_1 index e7bebf8..2aca0b0 100644 --- a/workflow-help.quarto_ipynb_1 +++ b/workflow-help.quarto_ipynb_1 @@ -35,7 +35,7 @@ "\n", "- First, you need to make your code reproducible. This means that you need to capture everything, i.e., include any packages you used and create all necessary objects. The easiest way to make sure you've done this is to use the [**watermark**](https://github.com/rasbt/watermark) package alongside whatever else you are doing:" ], - "id": "22b3f9e0" + "id": "3a1b5fcb" }, { "cell_type": "code", @@ -48,7 +48,7 @@ "print(watermark())\n", "print(watermark(iversions=True, globals_=globals()))" ], - "id": "a119501b", + "id": "06cb8178", "execution_count": null, "outputs": [] }, @@ -72,7 +72,7 @@ "\n", "- **Data**: as others won't be able to easily download the data you're working with, it's often best to create a small amount of data from code that still have the same problem as you're finding with your actual data. Between **numpy** and **pandas**, it's quite easy to generate data from code; here's an example:" ], - "id": "c4ac60b4" + "id": "2f7904c5" }, { "cell_type": "code", @@ -87,7 +87,7 @@ "df[\"random_normal\"] = np.random.normal(size=6)\n", "df" ], - "id": "d1e4562c", + "id": "1208648b", "execution_count": null, "outputs": [] }, @@ -99,7 +99,7 @@ "\n", "Finish by checking that you have actually made a reproducible example by starting a fresh Python session and copying and pasting your reprex in." ], - "id": "4b75e409" + "id": "a3f71a11" } ], "metadata": { diff --git a/workflow-packages-and-environments.quarto_ipynb_1 b/workflow-packages-and-environments.quarto_ipynb_1 index a5600ce..beb6dc7 100644 --- a/workflow-packages-and-environments.quarto_ipynb_1 +++ b/workflow-packages-and-environments.quarto_ipynb_1 @@ -104,7 +104,7 @@ "\n", "You may have noticed that a file called `pyproject.toml` has been created." ], - "id": "8b889898" + "id": "55930d3f" }, { "cell_type": "code", @@ -115,7 +115,7 @@ "\n", "print_json(data=toml.load(\"pyproject.toml\"))" ], - "id": "688f09f1", + "id": "1f3270eb", "execution_count": null, "outputs": [] }, @@ -133,7 +133,7 @@ "\n", "In the screenshot above, you can see the project-environment in two places: on the blue bar at the bottom of the screen, and (in 5), at the top right hand side of the interactive window. A similar top right indicator is present when you have a Jupyter Notebook open too." ], - "id": "148595b3" + "id": "9691960f" } ], "metadata": { From 93ffcd051bcf69a201ee3a7baaffbce891d41d67 Mon Sep 17 00:00:00 2001 From: Uchenna Ugoh Date: Fri, 24 Apr 2026 19:45:34 +0100 Subject: [PATCH 27/34] Refactor data-import and data-transform notebooks to replace pandas with polars - Updated import statements and function calls in data-import.ipynb to use polars instead of pandas. - Adjusted text descriptions to reflect the changes from pandas to polars, including installation instructions and function usage. - Cleaned up markdown formatting for better readability. - Removed unnecessary sections and streamlined explanations in data-transform.ipynb. --- _quarto.yml | 2 +- communicate-plots.ipynb | 189 ++++++++++++++++++++-------------------- data-import.ipynb | 116 ++++++++++-------------- data-transform.ipynb | 9 +- 4 files changed, 146 insertions(+), 170 deletions(-) diff --git a/_quarto.yml b/_quarto.yml index 51699c0..5c5e7f4 100644 --- a/_quarto.yml +++ b/_quarto.yml @@ -7,7 +7,7 @@ book: author: "The Py4DS Community" favicon: favicon.ico cover-image: logo.png - repo-url: https://github.com/aeturrell/python4DS + repo-url: https://github.com/datathink/python4DSpolars repo-branch: main repo-actions: [edit, issue] google-analytics: "G-LXJC37BJVX" diff --git a/communicate-plots.ipynb b/communicate-plots.ipynb index 8531e79..fc14db8 100644 --- a/communicate-plots.ipynb +++ b/communicate-plots.ipynb @@ -11,28 +11,24 @@ "\n", "In this chapter, you'll learn about using visualisation to communicate.\n", "\n", - "In @sec-exploratory-data-analysis, you learned how to use plots as tools for *exploration*.\n", + "In @sec-exploratory-data-analysis, you learned how to use plots as tools for _exploration_.\n", "When you make exploratory plots, you know—even before looking—which variables the plot will display.\n", "You made each plot for a purpose, quickly looked at it, and then moved on to the next plot.\n", "In the course of most analyses, you'll produce tens or hundreds of plots, most of which are immediately thrown away.\n", "\n", - "Now that you understand your data, you need to *communicate* your understanding to others.\n", + "Now that you understand your data, you need to _communicate_ your understanding to others.\n", "Your audience will likely not share your background knowledge and will not be deeply invested in the data. To help others quickly build up a good mental model of the data, you will need to invest considerable effort in making your plots as self-explanatory as possible. In this chapter, you'll learn some of the tools that **lets-plot** provides to do make charts tell a story.\n", "\n", "### Prerequisities\n", "\n", - "As ever, there are a plethora of options (and packages) for data visualisation using code. We're focusing on the declarative, \"grammar of graphics\" approach using **lets-plot** here, but advanced users looking for more complex graphics might wish to use an imperative library such as the excellent **matplotlib**. You should have both **lets-plot** and **pandas** installed. Once you have them installed, import them like so:" + "As ever, there are a plethora of options (and packages) for data visualisation using code. We're focusing on the declarative, \"grammar of graphics\" approach using **lets-plot** here, but advanced users looking for more complex graphics might wish to use an imperative library such as **Plotly Graph Objects**. You should have both **lets-plot** and **polars** installed. Once you have them installed, import them like so:\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "51a55374", - "metadata": { - "tags": [ - "remove-cell" - ] - }, + "id": "78eeea41", + "metadata": {}, "outputs": [], "source": [ "# remove cell\n", @@ -52,7 +48,7 @@ "outputs": [], "source": [ "import numpy as np\n", - "import pandas as pd\n", + "import polars as pl\n", "from lets_plot import *\n", "\n", "LetsPlot.setup_html()" @@ -65,7 +61,7 @@ "source": [ "## Labels, titles, and other contextual information\n", "\n", - "The easiest place to start when turning an exploratory graphic into an expository graphic is with good labels. Let's look at an example using the MPG (miles per gallon) data, which covers the fuel economy for 38 popular models of cars from 1999 to 2008." + "The easiest place to start when turning an exploratory graphic into an expository graphic is with good labels. Let's look at an example using the MPG (miles per gallon) data, which covers the fuel economy for 38 popular models of cars from 1999 to 2008.\n" ] }, { @@ -76,8 +72,9 @@ "outputs": [], "source": [ "# load the data\n", - "mpg = pd.read_csv(\n", - " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/mpg.csv\", index_col=0\n", + "mpg = pl.read_csv(\n", + " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/mpg.csv\",\n", + " row_index_offset=0,\n", ")" ] }, @@ -86,7 +83,7 @@ "id": "1813ab08", "metadata": {}, "source": [ - "We want to show fuel efficiency on the highway changes with engine displacement, in litres. The most basic chart we can do with these variables is:" + "We want to show fuel efficiency on the highway changes with engine displacement, in litres. The most basic chart we can do with these variables is:\n" ] }, { @@ -114,7 +111,7 @@ "- remove the y-axis label that is at an awkward viewing angle\n", "- add a caption with the source of the data\n", "\n", - "Putting this all in, we get:" + "Putting this all in, we get:\n" ] }, { @@ -143,9 +140,9 @@ "id": "2e28877a", "metadata": {}, "source": [ - "This is much clearer. It's easier to read, we know where the data come from, and we can see *why* we're being shown it too.\n", + "This is much clearer. It's easier to read, we know where the data come from, and we can see _why_ we're being shown it too.\n", "\n", - "But maybe we want a different message? You can flex depending on your needs, and some people prefer to have a rotated y-axis so that the subtitle can provide even more context:" + "But maybe we want a different message? You can flex depending on your needs, and some people prefer to have a rotated y-axis so that the subtitle can provide even more context:\n" ] }, { @@ -180,7 +177,7 @@ "1. Create one plot on the fuel economy data with customized `title`, `subtitle`, `caption`, `x`, `y`, and `color` labels.\n", "\n", "2. Recreate the following plot using the fuel economy data.\n", - " Note that both the colours and shapes of points vary by type of drive train." + " Note that both the colours and shapes of points vary by type of drive train.\n" ] }, { @@ -211,7 +208,7 @@ "id": "e10cdbd9", "metadata": {}, "source": [ - "3. Take an exploratory graphic that you've created in the last month, and add informative titles to make it easier for others to understand." + "3. Take an exploratory graphic that you've created in the last month, and add informative titles to make it easier for others to understand.\n" ] }, { @@ -229,7 +226,7 @@ "There are two possible sources of labels: ones that are part of the data, which we'll add with `geom_text()`; and ones that we add directly and manually as annotations using `geom_label()`.\n", "\n", "In the first case, you might have a data frame that contains labels.\n", - "In the following plot we pull out the cars with the highest engine size in each drive type and save their information as a new data frame called `label_info`. In creating it, we pick out the mean values of \"hwy\" by \"drv\" as the points to label—but we could do any aggregation we feel would work well on the chart." + "In the following plot we pull out the cars with the highest engine size in each drive type and save their information as a new data frame called `label_info`. In creating it, we pick out the mean values of \"hwy\" by \"drv\" as the points to label—but we could do any aggregation we feel would work well on the chart.\n" ] }, { @@ -245,11 +242,13 @@ " \"r\": \"rear-wheel drive\",\n", "}\n", "label_info = (\n", - " mpg.groupby(\"drv\")\n", - " .agg({\"hwy\": \"mean\", \"displ\": \"mean\"})\n", - " .reset_index()\n", - " .assign(drive_type=lambda x: x[\"drv\"].map(mapping))\n", - " .round(2)\n", + " mpg.group_by(\"drv\")\n", + " .agg(pl.mean(\"hwy\").round(2), pl.mean(\"displ\").round(2))\n", + " .with_columns(\n", + " pl.col(\"drv\")\n", + " .replace_strict(mapping, return_dtype=pl.String)\n", + " .alias(\"drive_type\")\n", + " )\n", ")\n", "label_info" ] @@ -259,7 +258,7 @@ "id": "93a136fe", "metadata": {}, "source": [ - "Then, we use this new data frame to directly label the three groups to replace the legend with labels placed directly on the plot. Using the fontface and size arguments we can customize the look of the text labels. They’re larger than the rest of the text on the plot and bolded. (`theme(legend.position = \"none\")` turns all the legends off — we’ll talk about it more shortly.)" + "Then, we use this new data frame to directly label the three groups to replace the legend with labels placed directly on the plot. Using the fontface and size arguments we can customize the look of the text labels. They’re larger than the rest of the text on the plot and bolded. (`theme(legend.position = \"none\")` turns all the legends off — we’ll talk about it more shortly.)\n" ] }, { @@ -292,8 +291,7 @@ "source": [ "Note the use of `hjust` (horizontal justification) and `vjust` (vertical justification) to control the alignment of the label.\n", "\n", - "\n", - "The second of the two methods we're looking at is `geom_label()`. This has two modes: in the first, it works like `geom_text()` but with a box around the text, like so:" + "The second of the two methods we're looking at is `geom_label()`. This has two modes: in the first, it works like `geom_text()` but with a box around the text, like so:\n" ] }, { @@ -303,7 +301,9 @@ "metadata": {}, "outputs": [], "source": [ - "potential_outliers = mpg.query(\"hwy > 40 | (hwy > 20 & displ > 5)\")\n", + "potential_outliers = mpg.filter(\n", + " (pl.col(\"hwy\") > 40) | ((pl.col(\"hwy\") > 20) & (pl.col(\"displ\") > 5))\n", + ")\n", "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", " + geom_point(color=\"black\")\n", @@ -331,7 +331,7 @@ "id": "38b69dcf", "metadata": {}, "source": [ - "The second method is generally useful for adding either a single or several annotations to a plot, like so:" + "The second method is generally useful for adding either a single or several annotations to a plot, like so:\n" ] }, { @@ -368,7 +368,7 @@ "id": "0720e7eb", "metadata": {}, "source": [ - "Annotation is a powerful tool for communicating main takeaways and interesting features of your visualisations. The only limit is your imagination (and your patience with positioning annotations to be aesthetically pleasing)!" + "Annotation is a powerful tool for communicating main takeaways and interesting features of your visualisations. The only limit is your imagination (and your patience with positioning annotations to be aesthetically pleasing)!\n" ] }, { @@ -379,15 +379,15 @@ "Remember, in addition to `geom_text()` and `geom_label()`, you have many other geoms in **lets-plot** available to help annotate your plot.\n", "A couple ideas:\n", "\n", - "- Use `geom_hline()` and `geom_vline()` to add reference lines.\n", - " We often make them thick (`size = 2`) and grey (`color = gray`), and draw them underneath the primary data layer.\n", - " That makes them easy to see, without drawing attention away from the data.\n", + "- Use `geom_hline()` and `geom_vline()` to add reference lines.\n", + " We often make them thick (`size = 2`) and grey (`color = gray`), and draw them underneath the primary data layer.\n", + " That makes them easy to see, without drawing attention away from the data.\n", "\n", - "- Use `geom_rect()` to draw a rectangle around points of interest.\n", - " The boundaries of the rectangle are defined by aesthetics `xmin`, `xmax`, `ymin`, `ymax`.\n", + "- Use `geom_rect()` to draw a rectangle around points of interest.\n", + " The boundaries of the rectangle are defined by aesthetics `xmin`, `xmax`, `ymin`, `ymax`.\n", "\n", - "- You already saw the use of `geom_segment()` with the `arrow` argument to draw attention to a point with an arrow.\n", - " Use aesthetics `x` and `y` to define the starting location, and `xend` and `yend` to define the end location.\n" + "- You already saw the use of `geom_segment()` with the `arrow` argument to draw attention to a point with an arrow.\n", + " Use aesthetics `x` and `y` to define the starting location, and `xend` and `yend` to define the end location.\n" ] }, { @@ -445,7 +445,7 @@ " scale_y_continuous() +\n", " scale_color_discrete()\n", ")\n", - "```" + "```\n" ] }, { @@ -461,11 +461,11 @@ "The default scales have been carefully chosen to do a good job for a wide range of inputs.\n", "Nevertheless, you might want to override the defaults for two reasons:\n", "\n", - "- You might want to tweak some of the parameters of the default scale.\n", - " This allows you to do things like change the breaks on the axes, or the key labels on the legend.\n", + "- You might want to tweak some of the parameters of the default scale.\n", + " This allows you to do things like change the breaks on the axes, or the key labels on the legend.\n", "\n", - "- You might want to replace the scale altogether, and use a completely different algorithm.\n", - " Often you can do better than the default because you know more about the data.\n" + "- You might want to replace the scale altogether, and use a completely different algorithm.\n", + " Often you can do better than the default because you know more about the data.\n" ] }, { @@ -478,8 +478,8 @@ "Collectively axes and legends get the somewhat confusing name **guides** in **lets-plot**. Axes are used for x and y aesthetics; legends are used for everything else.\n", "\n", "There are two primary arguments that affect the appearance of the ticks on the axes and the keys on the legend: `breaks` and `labels`.\n", - "Breaks controls the position of the ticks, or the values associated with the keys. If you like, the breaks *are* the ticks.\n", - "Labels controls the text label associated with each tick/key. We might more accurately call these *tick labels*.\n", + "Breaks controls the position of the ticks, or the values associated with the keys. If you like, the breaks _are_ the ticks.\n", + "Labels controls the text label associated with each tick/key. We might more accurately call these _tick labels_.\n", "The most common use of `breaks` is to override the default choice:\n" ] }, @@ -534,7 +534,7 @@ "- the .2 says use two significant digits\n", "- the s says, use the Système International (SI)\n", "\n", - "There are a wealth of alternative options for formatting—it's best to use the [helpful page on formatting](https://lets-plot.org/pages/formats.html) in the documentation of **lets-plot** to find out more." + "There are a wealth of alternative options for formatting—it's best to use the [helpful page on formatting](https://lets-plot.org/pages/formats.html) in the documentation of **lets-plot** to find out more.\n" ] }, { @@ -544,17 +544,16 @@ "metadata": {}, "outputs": [], "source": [ - "diamonds = pd.read_csv(\n", + "diamonds = pl.read_csv(\n", " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/diamonds.csv\",\n", - " index_col=0,\n", + " row_index_offset=0,\n", ")\n", - "diamonds[\"cut\"] = diamonds[\"cut\"].astype(\n", - " pd.CategoricalDtype(\n", - " categories=[\"Fair\", \"Good\", \"Very Good\", \"Premium\", \"Ideal\"], ordered=True\n", - " )\n", + "\n", + "diamonds = diamonds.with_columns(\n", + " pl.col(\"cut\").cast(pl.Enum([\"Fair\", \"Good\", \"Very Good\", \"Premium\", \"Ideal\"]))\n", ")\n", - "diamonds[\"color\"] = diamonds[\"color\"].astype(\n", - " pd.CategoricalDtype(categories=[\"D\", \"E\", \"F\", \"G\", \"H\", \"I\", \"J\"], ordered=True)\n", + "diamonds = diamonds.with_columns(\n", + " pl.col(\"color\").cast(pl.Enum([\"D\", \"E\", \"F\", \"G\", \"H\", \"I\", \"J\"]))\n", ")" ] }, @@ -578,7 +577,7 @@ "id": "6f2edc1b", "metadata": {}, "source": [ - "Another use of breaks is when you have relatively few data points and want to highlight exactly where the observations occur. For example, take this plot that shows when each US president started and ended their term." + "Another use of breaks is when you have relatively few data points and want to highlight exactly where the observations occur. For example, take this plot that shows when each US president started and ended their term.\n" ] }, { @@ -588,19 +587,22 @@ "metadata": {}, "outputs": [], "source": [ - "presidential = pd.read_csv(\n", + "presidential = pl.read_csv(\n", " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/presidential.csv\",\n", - " index_col=0,\n", + " row_index_offset=0,\n", + ")\n", + "presidential = presidential.with_columns(\n", + " pl.col(\"start\").str.strptime(pl.Date, \"%Y-%m-%d\").dt.year(),\n", + " pl.col(\"end\").str.strptime(pl.Date, \"%Y-%m-%d\").dt.year(),\n", + " (pl.col(\"rownames\") + 33).alias(\"id\"),\n", ")\n", - "presidential = presidential.astype({\"start\": \"datetime64[ns]\", \"end\": \"datetime64[ns]\"})\n", - "presidential[\"id\"] = 33 + presidential.index\n", "presidential.head()" ] }, { "cell_type": "code", "execution_count": null, - "id": "cd2cc430", + "id": "7d88976d", "metadata": {}, "outputs": [], "source": [ @@ -624,7 +626,7 @@ "\n", "To control the overall position of the legend, you need to use a `theme()` setting.\n", "We'll come back to themes at the end of the chapter, but in brief, they control the non-data parts of the plot.\n", - "The theme setting `legend.position` controls where the legend is drawn, and to demonstrate this we'll use `gggrid()` to arrange all of the plots." + "The theme setting `legend.position` controls where the legend is drawn, and to demonstrate this we'll use `gggrid()` to arrange all of the plots.\n" ] }, { @@ -651,7 +653,7 @@ "source": [ "If your plot is short and wide, place the legend at the top or bottom, and if it's tall and narrow, place the legend at the left or right. You can also use `legend_position = \"none\"` to suppress the display of the legend altogether.\n", "\n", - "To control the display of individual legends, use `guides()` along with `guide_legend()` or `guide_colorbar()`." + "To control the display of individual legends, use `guides()` along with `guide_legend()` or `guide_colorbar()`.\n" ] }, { @@ -659,7 +661,6 @@ "id": "f27913c7", "metadata": {}, "source": [ - "\n", "### Replacing a scale\n", "\n", "Instead of just tweaking the details a little, you can instead replace the scale altogether.\n", @@ -667,7 +668,7 @@ "Fortunately, the same principles apply to all the other aesthetics, so once you've mastered position and colour, you'll be able to quickly pick up other scale replacements.\n", "\n", "It's very useful to plot transformations of your variable.\n", - "For example, it's easier to see the precise relationship between `carat` and `price` if we log transform them. The way to do this is by using an `apply()` function on the data that gets sent to `ggplot`:" + "For example, it's easier to see the precise relationship between `carat` and `price` if we log transform them. The way to do this is by using an `apply()` function on the data that gets sent to `ggplot`:\n" ] }, { @@ -679,7 +680,7 @@ "source": [ "(\n", " ggplot(\n", - " diamonds.apply({\"carat\": np.log10, \"price\": np.log10}),\n", + " diamonds.with_columns(pl.col(\"carat\").log10(), pl.col(\"price\").log10()),\n", " aes(x=\"carat\", y=\"price\"),\n", " )\n", " + geom_bin2d()\n", @@ -691,7 +692,7 @@ "id": "f19dbbec", "metadata": {}, "source": [ - "However, the disadvantage of this transformation is that the axes are now mislabelled with the original values, making it hard to interpret the plot. Instead of doing the transformation in the aesthetic mapping, we can instead do it with the scale. This is visually identical, except the axes are labelled on the original data scale." + "However, the disadvantage of this transformation is that the axes are now mislabelled with the original values, making it hard to interpret the plot. Instead of doing the transformation in the aesthetic mapping, we can instead do it with the scale. This is visually identical, except the axes are labelled on the original data scale.\n" ] }, { @@ -714,7 +715,7 @@ "id": "4402c4de", "metadata": {}, "source": [ - "Another scale that is frequently customised is colour. The default categorical scale picks colors that are evenly spaced around the color wheel. Useful alternatives are the ColorBrewer scales which have been hand tuned to work better for people with common types of colour blindness. The two plots below look similar, but there is enough difference in the shades of red and green that the dots on the right can be distinguished even by people with red-green colour blindness." + "Another scale that is frequently customised is colour. The default categorical scale picks colors that are evenly spaced around the color wheel. Useful alternatives are the ColorBrewer scales which have been hand tuned to work better for people with common types of colour blindness. The two plots below look similar, but there is enough difference in the shades of red and green that the dots on the right can be distinguished even by people with red-green colour blindness.\n" ] }, { @@ -748,7 +749,7 @@ "source": [ "Don't forget simpler techniques for improving accessibility.\n", "If there are just a few colors, you can add a redundant shape mapping.\n", - "This will also help ensure your plot is interpretable in black and white." + "This will also help ensure your plot is interpretable in black and white.\n" ] }, { @@ -756,7 +757,7 @@ "id": "253af5a4", "metadata": {}, "source": [ - "The ColorBrewer scales are documented online at . The sequential (top) and diverging (bottom) palettes are particularly useful if your categorical values are ordered, or have a \"middle\". This often arises if you've used `pd.cut()` to make a continuous variable into a categorical variable." + "The ColorBrewer scales are documented online at . The sequential (top) and diverging (bottom) palettes are particularly useful if your categorical values are ordered, or have a \"middle\". This often arises if you've used `pd.cut()` to make a continuous variable into a categorical variable.\n" ] }, { @@ -905,7 +906,7 @@ "id": "c32c2237", "metadata": {}, "source": [ - "When you have a predefined mapping between values and colours, use `scale_color_manual()`. For example, if we map presidential party to colour, we want to use the standard mapping of red for Republicans and blue for Democrats. One approach for assigning these colors is using hex colour codes:" + "When you have a predefined mapping between values and colours, use `scale_color_manual()`. For example, if we map presidential party to colour, we want to use the standard mapping of red for Republicans and blue for Democrats. One approach for assigning these colors is using hex colour codes:\n" ] }, { @@ -915,13 +916,13 @@ "metadata": {}, "outputs": [], "source": [ - "mini_presid = presidential.iloc[5:, :]\n", + "mini_presid = presidential.slice(5)\n", "\n", "(\n", " ggplot(mini_presid, aes(x=\"start\", y=\"id\", color=\"party\"))\n", " + geom_point(size=3)\n", " + geom_segment(aes(xend=\"end\", yend=\"id\"), size=1)\n", - " + scale_x_datetime(breaks=mini_presid[\"start\"], format=\"%Y\")\n", + " + scale_x_datetime(breaks=mini_presid[\"start\"].to_list(), format=\"%Y\")\n", " + scale_color_manual(values=[\"#00AEF3\", \"#E81B23\"], name=\"party\")\n", ")" ] @@ -936,7 +937,7 @@ "For continuous colour, you can use the built-in `scale_color_gradient()` or `scale_fill_gradient()`.\n", "If you have a diverging scale, you can use `scale_color_gradient2()`. That allows you to give, for example, positive and negative values different colors. That's sometimes also useful if you want to distinguish points above or below the mean.\n", "\n", - "Another option is to use the viridis, magma, inferno, and plasma color scales developed for the extremely powerful imperative Python plotting package **[matplotlib](https://matplotlib.org/)**. The designers, Nathaniel Smith and Stéfan van der Walt, carefully tailored continuous color schemes that are perceptible to people with various forms of color blindness as well as perceptually uniform in both color and black and white. These scales are available as palettes in *lets-plot*. Here's an example using the continuous version of viridis (we'll generate some random data first):" + "Another option is to use the viridis, magma, inferno, and plasma color scales developed for the extremely powerful imperative Python plotting package **[matplotlib](https://matplotlib.org/)**. The designers, Nathaniel Smith and Stéfan van der Walt, carefully tailored continuous color schemes that are perceptible to people with various forms of color blindness as well as perceptually uniform in both color and black and white. These scales are available as palettes in _lets-plot_. Here's an example using the continuous version of viridis (we'll generate some random data first):\n" ] }, { @@ -947,7 +948,7 @@ "outputs": [], "source": [ "prng = np.random.default_rng(1837) # prng=probabilistic random number generator\n", - "df_rnd = pd.DataFrame(prng.standard_normal((1000, 2)), columns=[\"x\", \"y\"])\n", + "df_rnd = pl.DataFrame(prng.standard_normal((1000, 2)), schema=[\"x\", \"y\"])\n", "(\n", " ggplot(df_rnd, aes(x=\"x\", y=\"y\"))\n", " + geom_bin2d()\n", @@ -1002,7 +1003,7 @@ ")\n", "\n", "(\n", - " ggplot(mpg.loc[mpg_condition], aes(x=\"displ\", y=\"hwy\"))\n", + " ggplot(mpg.filter(mpg_condition), aes(x=\"displ\", y=\"hwy\"))\n", " + geom_point(aes(color=\"drv\"))\n", " + geom_smooth(method=\"loess\")\n", ")" @@ -1015,7 +1016,7 @@ "source": [ "Let's compare these to the two plots below where the first plot sets the `limits` on individual scales and the second plot sets them in `coord_cartesian()`.\n", "We can see that reducing the limits is equivalent to subsetting the data.\n", - "Therefore, to zoom in on a region of the plot, it's generally best to use `coord_cartesian()`." + "Therefore, to zoom in on a region of the plot, it's generally best to use `coord_cartesian()`.\n" ] }, { @@ -1054,8 +1055,8 @@ "id": "5d1fc3ee", "metadata": {}, "source": [ - "On the other hand, setting the `limits` on individual scales is generally more useful if you want to *expand* the limits, e.g., to match scales across different plots.\n", - "For example, if we extract two classes of cars and plot them separately, it's difficult to compare the plots because all three scales (the x-axis, the y-axis, and the colour aesthetic) have different ranges." + "On the other hand, setting the `limits` on individual scales is generally more useful if you want to _expand_ the limits, e.g., to match scales across different plots.\n", + "For example, if we extract two classes of cars and plot them separately, it's difficult to compare the plots because all three scales (the x-axis, the y-axis, and the colour aesthetic) have different ranges.\n" ] }, { @@ -1065,8 +1066,8 @@ "metadata": {}, "outputs": [], "source": [ - "suv = mpg.loc[mpg[\"class\"] == \"suv\"]\n", - "compact = mpg.loc[mpg[\"class\"] == \"compact\"]\n", + "suv = mpg.filter(mpg[\"class\"] == \"suv\")\n", + "compact = mpg.filter(mpg[\"class\"] == \"compact\")\n", "(ggplot(suv, aes(x=\"displ\", y=\"hwy\", color=\"drv\")) + geom_point())" ] }, @@ -1095,9 +1096,9 @@ "metadata": {}, "outputs": [], "source": [ - "x_scale = scale_x_continuous(limits=mpg[\"displ\"].agg([\"max\", \"min\"]).tolist())\n", - "y_scale = scale_y_continuous(limits=mpg[\"hwy\"].agg([\"max\", \"min\"]).tolist())\n", - "col_scale = scale_color_discrete(limits=mpg[\"drv\"].unique())" + "x_scale = scale_x_continuous(limits=[mpg[\"displ\"].min(), mpg[\"displ\"].max()])\n", + "y_scale = scale_y_continuous(limits=[mpg[\"hwy\"].min(), mpg[\"hwy\"].max()])\n", + "col_scale = scale_color_discrete(limits=mpg[\"drv\"].unique().to_list)" ] }, { @@ -1152,11 +1153,11 @@ "\n", "2. Change the display of the presidential terms by:\n", "\n", - " a. Combining the two variants that customize colors and x axis breaks.\n", - " b. Improving the display of the y axis.\n", - " c. Labelling each term with the name of the president.\n", - " d. Adding informative plot labels.\n", - " e. Placing breaks every 4 years (this is trickier than it seems!).\n" + " a. Combining the two variants that customize colors and x axis breaks.\n", + " b. Improving the display of the y axis.\n", + " c. Labelling each term with the name of the president.\n", + " d. Adding informative plot labels.\n", + " e. Placing breaks every 4 years (this is trickier than it seems!).\n" ] }, { @@ -1191,7 +1192,7 @@ "source": [ "**lets-plot** includes several built-in themes that you can find [here](https://lets-plot.org/pages/api.html#predefined-themes). You can also create your own themes, if you are trying to match a particular corporate or journal style.\n", "\n", - "Here's an example of changing multiple `theme()` settings:" + "Here's an example of changing multiple `theme()` settings:\n" ] }, { @@ -1257,9 +1258,9 @@ "source": [ "## Saving plots to file\n", "\n", - "There are lots of output options to choose from to save your file to. Remember that, for graphics, *vector formats* are generally better than *raster formats*. In practice, this means saving plots in svg or pdf formats over jpg or png file formats. The svg format works in a lot of contexts (including Microsoft Word) and is a good default. To choose between formats, just supply the file extension and the file type will change automatically, eg \"chart.svg\" for svg or \"chart.png\" for png (thought note that raster formats often have extra options, like how many dots per inch to use).\n", + "There are lots of output options to choose from to save your file to. Remember that, for graphics, _vector formats_ are generally better than _raster formats_. In practice, this means saving plots in svg or pdf formats over jpg or png file formats. The svg format works in a lot of contexts (including Microsoft Word) and is a good default. To choose between formats, just supply the file extension and the file type will change automatically, eg \"chart.svg\" for svg or \"chart.png\" for png (thought note that raster formats often have extra options, like how many dots per inch to use).\n", "\n", - "Let's try this out using the figure we made in the previous exercise, `p1`. `path=\".\"` just drops the file in the current directory." + "Let's try this out using the figure we made in the previous exercise, `p1`. `path=\".\"` just drops the file in the current directory.\n" ] }, { @@ -1277,7 +1278,7 @@ "id": "7781794a", "metadata": {}, "source": [ - "To double check this has worked, let's use the terminal. We'll try the command `ls`, which lists everything in directory, and `grep *.svg` to pull out any files that end in `.svg` from what is returned by `ls`. These are strung together as commands by a `|`. (Note that the leading exclamation mark below just tells the software that builds this book to use the terminal.)" + "To double check this has worked, let's use the terminal. We'll try the command `ls`, which lists everything in directory, and `grep *.svg` to pull out any files that end in `.svg` from what is returned by `ls`. These are strung together as commands by a `|`. (Note that the leading exclamation mark below just tells the software that builds this book to use the terminal.)\n" ] }, { @@ -1319,7 +1320,7 @@ "\n", "While you've so far learned about how to make many different types of plots and how to customise them using a variety of techniques, we've barely scratched the surface of what you can create with **lets-plot**.\n", "\n", - "The best place to go for further information is the [**lets-plot** dcoumentation](https://lets-plot.org/)." + "The best place to go for further information is the [**lets-plot** dcoumentation](https://lets-plot.org/).\n" ] } ], @@ -1331,7 +1332,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "python4ds", "language": "python", "name": "python3" }, diff --git a/data-import.ipynb b/data-import.ipynb index 59363eb..c240a16 100644 --- a/data-import.ipynb +++ b/data-import.ipynb @@ -13,7 +13,7 @@ "\n", "### Prerequisites\n", "\n", - "You will need to have the **pandas** package installed. You'll need to ensure you have **pandas** installed. To do this, and to import **pandas** into your session, run" + "You will need to have the **pandas** package installed. You'll need to ensure you have **pandas** installed. To do this, and to import **pandas** into your session, run\n" ] }, { @@ -23,7 +23,7 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd" + "import polars as pl" ] }, { @@ -31,7 +31,7 @@ "id": "e29b7103", "metadata": {}, "source": [ - "If this command fails, you don't have **pandas** installed. Open up the terminal in Visual Studio Code (Terminal -> New Terminal), `cd` to the folder you are working in, and type in `uv add pandas`. Note that once **pandas** is installed, the convention is to import it into your Python session under the name `pd` by putting `import pandas as pd` at the top of your script." + "If this command fails, you don't have **polars** installed. Open up the terminal in Visual Studio Code (Terminal -> New Terminal), `cd` to the folder you are working in, and type in `uv add polars`. Note that once **polars** is installed, the convention is to import it into your Python session under the name `pl` by putting `import polars as pl` at the top of your script.\n" ] }, { @@ -43,15 +43,13 @@ "\n", "There are a huge range of input and output formats available in **pandas**: Stata (.dta), Excel (.xls, .xlsx), csv, tsv, big data formats (HDF5, parquet), JSON, SAS, SPSS, SQL, and more; there's a [full list](https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html) of formats available in the documentation.\n", "\n", - "![From the pandas documentation](https://pandas.pydata.org/pandas-docs/stable/_images/02_io_readwrite.svg)\n", - "\n", - "While **pandas** has a huge number of ways to read data in and load it into your Python session, here we'll focus on the humble plain-text table file; for example csv (comma separated values) and tsv (tab separated values).\n", + "While **polars** has a huge number of ways to read data in and load it into your Python session, here we'll focus on the humble plain-text table file; for example csv (comma separated values) and tsv (tab separated values).\n", "\n", "### Reading data from a file\n", "\n", - "All of the power needed to open plain-text table files is contained in a single function, `pd.read_csv()`. It takes numerous arguments but the two most important are the (unnamed) first one, which gives the path to the data, and `sep=` (a keyword argument) that tells **pandas** whether to expect values to be separated by commas or tabs or another character; however, if you leave this field blank, **pandas** will guess for you. To see the full set of arguments, run `help(pd.read_csv)`.\n", + "All of the power needed to open plain-text table files is contained in a single function, `pl.read_csv()`. It takes numerous arguments but the two most important are the (unnamed) first one, which gives the path to the data, and `seperator=` (a keyword argument) that tells **polars** whether to expect values to be separated by commas or tabs or another character; however, if you leave this field blank, **polars** will guess for you. To see the full set of arguments, run `help(pl.read_csv)`.\n", "\n", - "Here is what a simple CSV file with a row for column names (also commonly referred to as the header row) and six rows of data looks like (using the terminal):" + "Here is what a simple CSV file with a row for column names (also commonly referred to as the header row) and six rows of data looks like (using the terminal):\n" ] }, { @@ -69,7 +67,7 @@ "id": "c7352c95", "metadata": {}, "source": [ - "Note that this is a CSV file, so the values are separated by commas. Now let's load this into a **pandas** dataframe in Python:" + "Note that this is a CSV file, so the values are separated by commas. Now let's load this into a **pandas** dataframe in Python:\n" ] }, { @@ -79,7 +77,7 @@ "metadata": {}, "outputs": [], "source": [ - "students = pd.read_csv(\"data/students.csv\")\n", + "students = pl.read_csv(\"data/students.csv\")\n", "students" ] }, @@ -96,7 +94,7 @@ "os.getcwd() # get current working directory (cwd)\n", "```\n", "\n", - "Say this comes back with 'python4DS', then your downloaded data should be in 'python4DS/data/students.csv'." + "Say this comes back with 'python4DS', then your downloaded data should be in 'python4DS/data/students.csv'.\n" ] }, { @@ -104,17 +102,17 @@ "id": "2f3979fc", "metadata": {}, "source": [ - "The first argument to `read_csv()` was the path to the data, and **pandas** guessed that this file uses commas as the separator.\n", + "The first argument to `read_csv()` was the path to the data, and **polars** guessed that this file uses commas as the separator.\n", "\n", - "The read CSV function automatically creates a new *index* (which is just the position of each row) and takes the top line of data as the *header* or *column names*. But you may wish to tweak this behaviour, in several ways.\n", + "The read CSV function automatically creates a new _index_ (which is just the position of each row) and takes the top line of data as the _header_ or _column names_. But you may wish to tweak this behaviour, in several ways.\n", "\n", - "1. Sometimes there are a few lines of metadata at the top of the file. You can use `skiprows=n` to skip the first `n` lines, eg `pd.read_csv(\"data/students.csv\", skiprows=2)`.\n", + "1. Sometimes there are a few lines of metadata at the top of the file. You can use `skip_rows=n` to skip the first `n` lines, eg `pl.read_csv(\"data/students.csv\", skip_rows=2)`.\n", "\n", - "2. The data might not have column names. You can use `names = ` a list to tell `read_csv()` to use a different option for the column names. For example, `pd.read_csv(\"data/students.csv\", names=range(5))` would put the numbers 0 to 4 in as the column names.\n", + "2. The data might not have column names. You can use `columns= ` a list to tell `read_csv()` to use a different option for the column names. For example, `pl.read_csv(\"data/students.csv\", columns=range(5))` would put the numbers 0 to 4 in as the column names.\n", "\n", - "3. You may wish to change which column is used as the index. The default behaviour is to create an index, but for this data we see that there already is an ID column we could use. To do this, use the `index_col=` argument, for example `pd.read_csv(\"data/students.csv\", index_col=0)`.\n", + "3. You may wish to inset a column to be used as a row index. For this data we see that there already is an ID column we could use. To do this, use the `row_index_offset=` argument, for example `pd.read_csv(\"data/students.csv\", row_index_offset=0)`.\n", "\n", - "This is all you need to know to read \\~75% of CSV files that you'll encounter in practice. Reading tab separated files and fixed width files is done with the same function." + "This is all you need to know to read \\~75% of CSV files that you'll encounter in practice. Reading tab separated files and fixed width files is done with the same function.\n" ] }, { @@ -130,7 +128,7 @@ "\n", "You might consider renaming them one by one with `.rename()` or you might use a convenience function from another package to clean them and turn them all into snake case at once. We will make use of the **skimpy** package to do this. Install it by running `uv add skimpy` in the terminal.\n", "\n", - "From **skimpy**, we will use the `clean_columns()` function; this takes in a data frame and returns a data frame with variable names converted to snake case." + "From **skimpy**, we will use the `clean_columns()` function; this takes in a data frame and returns a data frame with variable names converted to snake case.\n" ] }, { @@ -151,26 +149,7 @@ "id": "69c8a260", "metadata": {}, "source": [ - "Another common task after reading in data is to consider variable types. In the `favourite_food` column, there are a bunch of food items and then the value `NaN`, which has been read in as a floating point number rather than a missing string. We can solve this by casting that column to explicitly be composed of strings:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b4dad370", - "metadata": {}, - "outputs": [], - "source": [ - "students[\"favourite_food\"] = students[\"favourite_food\"].astype(\"string\")\n", - "students" - ] - }, - { - "cell_type": "markdown", - "id": "e85148b3", - "metadata": {}, - "source": [ - "Similarly, `\"age\"` has mixed data types: string and integer! Let's map the 'five' into the number five." + "Another common task after reading in data is to consider variable types. In the `age` column, there are mixed data types: string and integer!, which has been read in as a string. We can solve this by mapping the 'five' into the number five, then casting that column to explicitly be composed of integers:\n" ] }, { @@ -180,7 +159,7 @@ "metadata": {}, "outputs": [], "source": [ - "students[\"age\"] = students[\"age\"].replace(\"five\", 5)\n", + "students = students.with_columns(pl.col(\"age\").replace(\"five\", 5).cast(pl.Int64))\n", "students[\"age\"]" ] }, @@ -189,7 +168,7 @@ "id": "719f90cc", "metadata": {}, "source": [ - "In a moment, we will turn this into a column of integers too." + "In a moment, we will turn this into a column of integers too.\n" ] }, { @@ -197,7 +176,7 @@ "id": "e5c2b01e", "metadata": {}, "source": [ - "Another example where the data type is wrong is `meal_type`. This is a categorical variable with a known set of possible values. **pandas** has a special data type for these:" + "Another example where the data type is wrong is `meal_plan`. This is a categorical variable with a known set of possible values. **polars** has a special data type for these:\n" ] }, { @@ -207,7 +186,7 @@ "metadata": {}, "outputs": [], "source": [ - "students[\"meal_plan\"] = students[\"meal_plan\"].astype(\"category\")\n", + "students = students.with_columns(pl.col(\"meal_plan\").cast(pl.Categorical))\n", "students[\"meal_plan\"]" ] }, @@ -216,9 +195,9 @@ "id": "1e2a32a8", "metadata": {}, "source": [ - "Note that the values in the `meal_type` variable has stayed exactly the same, but the type of variable has changed from the object to category.\n", + "Note that the values in the `meal_plan` variable has stayed exactly the same, but the type of variable has changed from the object to category.\n", "\n", - "It is a bit tedious to have to go through columns one-by-one as single line assignments to apply type. An alternative is to pass a dictionary that maps column names into types, like follows:" + "It is a bit tedious to have to go through columns one-by-one as single line assignments to apply type. An alternative is to pass a dictionary that maps column names into types, like follows:\n" ] }, { @@ -228,8 +207,10 @@ "metadata": {}, "outputs": [], "source": [ - "students = students.astype({\"student_id\": \"int\", \"full_name\": \"string\", \"age\": \"int\"})\n", - "students.info()" + "students = students.cast(\n", + " {\"student_id\": pl.Int32, \"full_name\": pl.String, \"age\": pl.Int64}\n", + ")\n", + "students.schema" ] }, { @@ -239,7 +220,7 @@ "source": [ "### Exercises\n", "\n", - "1. What function would you use to read a file where fields were separated with \"\\|\"?" + "1. What function would you use to read a file where fields were separated with \"\\|\"?\n" ] }, { @@ -247,7 +228,7 @@ "id": "3fc7dbd6", "metadata": {}, "source": [ - "## Reading data from multiple files" + "## Reading data from multiple files\n" ] }, { @@ -257,7 +238,7 @@ "source": [ "Sometimes your data is split across multiple files instead of being contained in a single file. For example, you might have sales data for multiple months, with each month's data in a separate file: `01-sales.csv` for January, `02-sales.csv` for February, and `03-sales.csv` for March.\n", "\n", - "With `pd.read_csv()` you can read these data in one-by-one and then stack them on top of each other in a single data frame using the `pd.concat()` function. This looks like:\n" + "With `pl.read_csv()` you can read these data in one-by-one and then stack them on top of each other in a single data frame using the `pl.concat()` function. This looks like:\n" ] }, { @@ -268,10 +249,10 @@ "outputs": [], "source": [ "list_of_dataframes = [\n", - " pd.read_csv(x)\n", + " pl.read_csv(x)\n", " for x in [\"data/01-sales.csv\", \"data/02-sales.csv\", \"data/03-sales.csv\"]\n", "]\n", - "sales_files = pd.concat(list_of_dataframes)\n", + "sales_files = pl.concat(list_of_dataframes)\n", "sales_files" ] }, @@ -280,7 +261,7 @@ "id": "ea3904ca", "metadata": {}, "source": [ - "If you have many files you want to read in, it can get cumbersome to write out their names as a list. Instead, you can use the **glob** package (which is built in to Python) to find the files for you by matching a pattern in the file names. Note that there may be other CSV files in the directory data/, so here we specified `\"*-sales.csv\"` to ensure we only get those files that include the word sales. Here, `\"*\"` plays the role of a wildcard: it represents any series of characters." + "If you have many files you want to read in, it can get cumbersome to write out their names as a list. Instead, you can use the **glob** package (which is built in to Python) to find the files for you by matching a pattern in the file names. Note that there may be other CSV files in the directory data/, so here we specified `\"*-sales.csv\"` to ensure we only get those files that include the word sales. Here, `\"*\"` plays the role of a wildcard: it represents any series of characters.\n" ] }, { @@ -295,7 +276,7 @@ "list_of_csvs = glob.glob(\"data/*-sales.csv\")\n", "print(\"List of csvs is:\")\n", "print(list_of_csvs, \"\\n\")\n", - "sales_files = pd.concat([pd.read_csv(x) for x in list_of_csvs])\n", + "sales_files = pl.concat([pl.read_csv(x) for x in list_of_csvs])\n", "sales_files" ] }, @@ -306,9 +287,9 @@ "source": [ "## Writing to a file\n", "\n", - "Just as the typical pattern for reading files is `pd.read_FILETYPE()`, where filetype can be, for example, CSV, all of the ways of writing **pandas** data frames to disk have the pattern `DATAFRAME.to_FILETYPE()`. So to write our sales data to a CSV file, the code will be `sales_files.to_csv(FILEPATH)`, where filepath is the location plus name of the file you want to write to.\n", + "Just as the typical pattern for reading files is `pd.read_FILETYPE()`, where filetype can be, for example, CSV, all of the ways of writing **polars** data frames to disk have the pattern `DATAFRAME.write_FILETYPE()`. So to write our sales data to a CSV file, the code will be `sales_files.write_csv(FILEPATH)`, where filepath is the location plus name of the file you want to write to.\n", "\n", - "Let's see an example of writing data to file using our students data that we already did such good work on setting the data types for:" + "Let's see an example of writing data to file using our students data that we already did such good work on setting the data types for:\n" ] }, { @@ -318,7 +299,7 @@ "metadata": {}, "outputs": [], "source": [ - "students.to_csv(\"data/students-clean.csv\")" + "students.write_csv(\"data/students-clean.csv\")" ] }, { @@ -326,7 +307,7 @@ "id": "9c92b016", "metadata": {}, "source": [ - "Now let's read it back in and check the info on data types:" + "Now let's read it back in and check the info on data types:\n" ] }, { @@ -336,7 +317,7 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_csv(\"data/students-clean.csv\").info()" + "pl.read_csv(\"data/students-clean.csv\").schema" ] }, { @@ -344,13 +325,13 @@ "id": "164cd1c0", "metadata": {}, "source": [ - "Notice anything? We lost a lot of the nice data type work we did! While **pandas** guessed that some columns are integers, we lost the string and categorical variables. The reason for this is that plain text files are not able to carry any contextual information (though **pandas** will guess some column data types).\n", + "Notice anything? We lost a lot of the nice data type work we did! While **polars** guessed that some columns are integers, we lost the string and categorical variables. The reason for this is that plain text files are not able to carry any contextual information (though **polars** will guess some column data types).\n", "\n", - "If you want to save data in a file and have it remember the data types, you'll need to use a different data format. For temporary storage, we recommend using the *feather* format as it is very fast and interoperable with other programming languages. Interoperability is a good reason to avoid language-specific file formats such as Stata's .dta, R's .rds, and Python's .pickle.\n", + "If you want to save data in a file and have it remember the data types, you'll need to use a different data format. For temporary storage, we recommend using the _feather_ format as it is very fast and interoperable with other programming languages. Interoperability is a good reason to avoid language-specific file formats such as Stata's .dta, R's .rds, and Python's .pickle.\n", "\n", "Note that the feather format has an additional dependency in the form of a package called **pyarrow**. To install it, run `uv add pyarrow` in a terminal window.\n", "\n", - "Here's an example of writing to a feather file:" + "Here's an example of writing to a feather file:\n" ] }, { @@ -360,7 +341,7 @@ "metadata": {}, "outputs": [], "source": [ - "students.to_feather(\"data/students-clean.feather\")" + "students.write_ipc(\"data/students-clean.feather\")" ] }, { @@ -368,7 +349,7 @@ "id": "2119dd53", "metadata": {}, "source": [ - "Now let's re-open that feather file and take a look at the info attached to it." + "Now let's re-open that feather file and take a look at the info attached to it.\n" ] }, { @@ -378,7 +359,7 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_feather(\"data/students-clean.feather\").info()" + "pl.read_ipc(\"data/students-clean.feather\").schema" ] }, { @@ -390,7 +371,7 @@ "\n", "### Reading and Writing Other Data Formats\n", "\n", - "The image at the start of this chapter gives you a sense of what other formats are available but you can find a comprehensive list over at the [official **pandas** documentation on input and output](https://pandas.pydata.org/docs/user_guide/io.html)." + "You can find a comprehensive list of what formats are available over at the [official **polars** documentation on input and output](https://pandas.pydata.org/docs/user_guide/io.html).\n" ] }, { @@ -412,9 +393,6 @@ } ], "metadata": { - "interpreter": { - "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" - }, "jupytext": { "cell_metadata_filter": "-all", "encoding": "# -*- coding: utf-8 -*-", @@ -422,7 +400,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "python4ds", "language": "python", "name": "python3" }, diff --git a/data-transform.ipynb b/data-transform.ipynb index c20ac1c..83588d0 100644 --- a/data-transform.ipynb +++ b/data-transform.ipynb @@ -557,11 +557,7 @@ "source": [ "## Manipulating Columns\n", "\n", - "This section will show you how to apply various operations you may need to columns in your data frame.\n", - "\n", - "::: {.callout-note}\n", - "Some **pandas** operations can apply either to columns or rows, depending on the syntax used. For example, accessing values by position can be achieved in the same way for rows and columns via `.iloc` where to access the ith row you would use `df.iloc[i]` and to access the jth column you would use `df.iloc[:, j]` where `:` stands in for 'any row'.\n", - ":::\n" + "This section will show you how to apply various operations you may need to columns in your data frame.\n" ] }, { @@ -880,7 +876,8 @@ "metadata": {}, "source": [ "::: {.callout-tip}\n", - "A _lambda function_ is a small, anonymous function in Python that performs a single operation. It's a shorthand way to create a function without using `def`.\n" + "A _lambda function_ is a small, anonymous function in Python that performs a single operation. It's a shorthand way to create a function without using `def`.\n", + ":::\n" ] }, { From 3d8929c410fe17a808667c5077a8d912ecdf9904 Mon Sep 17 00:00:00 2001 From: Uchenna Ugoh Date: Fri, 24 Apr 2026 19:52:59 +0100 Subject: [PATCH 28/34] Refactor functions.ipynb to remove output displays and execution counts, enhancing clarity and consistency in code presentation. --- functions.ipynb | 191 ++++++++++++++++-------------------------------- 1 file changed, 63 insertions(+), 128 deletions(-) diff --git a/functions.ipynb b/functions.ipynb index 4a0032f..7d55cbc 100644 --- a/functions.ipynb +++ b/functions.ipynb @@ -31,7 +31,7 @@ "\n", "### Prerequisites\n", "\n", - "You will need the **polars** and **numpy** packages for this chapter." + "You will need the **polars** and **numpy** packages for this chapter.\n" ] }, { @@ -43,23 +43,15 @@ "\n", "A function has inputs, it performs its function, and it returns any outputs. Functions begin with a def keyword for ‘define a function’. It then has a name, followed by brackets, (), which may contain function arguments and function keyword arguments. This is followed by a colon. The body of the function is then indented relative to the left-most text. Function arguments are defined in brackets following the name, with different inputs separated by commas. Any outputs are given with the return keyword, again with different variables separated by commas.\n", "\n", - "Let's see a very simple example of a function with a single *argument* (or arg):" + "Let's see a very simple example of a function with a single _argument_ (or arg):\n" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "0450ad6c", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Hello Ada, and welcome!\n" - ] - } - ], + "outputs": [], "source": [ "def welcome_message(name):\n", " return f\"Hello {name}, and welcome!\"\n", @@ -76,24 +68,15 @@ "id": "08c06c6a", "metadata": {}, "source": [ - "One powerful feature of functions is that we can define defaults for the input arguments. These are called *keyword arguments* (or kwargs). Let's see that in action by defining a default value for `name`, along with multiple outputs--a hello message and a score." + "One powerful feature of functions is that we can define defaults for the input arguments. These are called _keyword arguments_ (or kwargs). Let's see that in action by defining a default value for `name`, along with multiple outputs--a hello message and a score.\n" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "dd49bee5", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "('Hello student', 4.8)\n", - "('Hello Ada', 4.8)\n" - ] - } - ], + "outputs": [], "source": [ "def score_message(score, name=\"student\"):\n", " \"\"\"This is a doc-string, a string describing a function.\n", @@ -124,43 +107,24 @@ "source": [ "::: {.callout-note title=\"Arguments and keyword arguments\"}\n", "\n", - "*arguments* are the variables that functions *always* need, so `a` and `b` in `def add(a, b): return a + b`. The function won't work without them! Function arguments are sometimes referred to as *args*.\n", + "_arguments_ are the variables that functions _always_ need, so `a` and `b` in `def add(a, b): return a + b`. The function won't work without them! Function arguments are sometimes referred to as _args_.\n", "\n", - "*Keyword arguments* are the variables that are optional for functions, so `c` in `def add(a, b, c=5): return a + b - c`. If you do not provide a value for `c` when calling the function, it will automatically revert to `c=5`. Keyword arguments are sometimes referred to as *kwargs*.\n", + "_Keyword arguments_ are the variables that are optional for functions, so `c` in `def add(a, b, c=5): return a + b - c`. If you do not provide a value for `c` when calling the function, it will automatically revert to `c=5`. Keyword arguments are sometimes referred to as _kwargs_.\n", ":::\n", "\n", - "\n", "::: {.callout-tip title=\"Exercise\"}\n", "What is the return type of a function with multiple return values separated by commas following the `return` statement?\n", ":::\n", "\n", - "In that last example, you'll notice that we added some text to the function. This is a doc-string, or documentation string. It's there to help users (and, most likely, future you) to understand what the function does. Let's see how this works in action by calling `help()` on the `score_message` function:" + "In that last example, you'll notice that we added some text to the function. This is a doc-string, or documentation string. It's there to help users (and, most likely, future you) to understand what the function does. Let's see how this works in action by calling `help()` on the `score_message` function:\n" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "50dfff24", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Help on function score_message in module __main__:\n", - "\n", - "score_message(score, name='student')\n", - " This is a doc-string, a string describing a function.\n", - " Args:\n", - " score (float): Raw score\n", - " name (str): Name of student\n", - " Returns:\n", - " str: A hello message.\n", - " float: A normalised score.\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "help(score_message)" ] @@ -178,7 +142,7 @@ "Write a doc-string for your function and call `help` on it.\n", ":::\n", "\n", - "To learn more about args and kwargs, check out these [short video tutorials](https://calmcode.io/args-kwargs/introduction.html)." + "To learn more about args and kwargs, check out these [short video tutorials](https://calmcode.io/args-kwargs/introduction.html).\n" ] }, { @@ -190,64 +154,37 @@ "\n", "You should consider writing a function whenever you've copied and pasted a block of code more than twice (i.e. you now have three copies of the same code).\n", "For example, take a look at this code.\n", - "What does it do?" + "What does it do?\n" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "43d10e40", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (10, 4)
abcd
f64f64f64f64
0.9140810.873650.4582790.785412
0.0738862.0167310.020991.0
1.00.7365450.3230560.98617
0.3479981.4784291.00.0
0.0284780.00.3285560.807754
0.01.3396160.2558630.763866
0.3223031.4192460.00.686972
0.4370661.2256020.2708620.951588
0.0916591.0990040.2531110.920187
0.376460.8695870.0566590.264292
" - ], - "text/plain": [ - "shape: (10, 4)\n", - "┌──────────┬──────────┬──────────┬──────────┐\n", - "│ a ┆ b ┆ c ┆ d │\n", - "│ --- ┆ --- ┆ --- ┆ --- │\n", - "│ f64 ┆ f64 ┆ f64 ┆ f64 │\n", - "╞══════════╪══════════╪══════════╪══════════╡\n", - "│ 0.914081 ┆ 0.87365 ┆ 0.458279 ┆ 0.785412 │\n", - "│ 0.073886 ┆ 2.016731 ┆ 0.02099 ┆ 1.0 │\n", - "│ 1.0 ┆ 0.736545 ┆ 0.323056 ┆ 0.98617 │\n", - "│ 0.347998 ┆ 1.478429 ┆ 1.0 ┆ 0.0 │\n", - "│ 0.028478 ┆ 0.0 ┆ 0.328556 ┆ 0.807754 │\n", - "│ 0.0 ┆ 1.339616 ┆ 0.255863 ┆ 0.763866 │\n", - "│ 0.322303 ┆ 1.419246 ┆ 0.0 ┆ 0.686972 │\n", - "│ 0.437066 ┆ 1.225602 ┆ 0.270862 ┆ 0.951588 │\n", - "│ 0.091659 ┆ 1.099004 ┆ 0.253111 ┆ 0.920187 │\n", - "│ 0.37646 ┆ 0.869587 ┆ 0.056659 ┆ 0.264292 │\n", - "└──────────┴──────────┴──────────┴──────────┘" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import numpy as np\n", "import polars as pl\n", "\n", "df = pl.DataFrame(np.random.normal(size=(10, 4)), schema=[\"a\", \"b\", \"c\", \"d\"])\n", "\n", - "df = df.with_columns([\n", - " ((pl.col(\"a\") - pl.col(\"a\").min()) / (pl.col(\"a\").max() - pl.col(\"a\").min())).alias(\"a\"),\n", - " ((pl.col(\"b\") - pl.col(\"b\").min()) / (pl.col(\"b\").max() - pl.col(\"a\").min())).alias(\"b\"),\n", - " ((pl.col(\"c\") - pl.col(\"c\").min()) / (pl.col(\"c\").max() - pl.col(\"c\").min())).alias(\"c\"),\n", - " ((pl.col(\"d\") - pl.col(\"d\").min()) / (pl.col(\"d\").max() - pl.col(\"d\").min())).alias(\"d\")\n", - "])\n", + "df = df.with_columns(\n", + " [\n", + " (\n", + " (pl.col(\"a\") - pl.col(\"a\").min()) / (pl.col(\"a\").max() - pl.col(\"a\").min())\n", + " ).alias(\"a\"),\n", + " (\n", + " (pl.col(\"b\") - pl.col(\"b\").min()) / (pl.col(\"b\").max() - pl.col(\"a\").min())\n", + " ).alias(\"b\"),\n", + " (\n", + " (pl.col(\"c\") - pl.col(\"c\").min()) / (pl.col(\"c\").max() - pl.col(\"c\").min())\n", + " ).alias(\"c\"),\n", + " (\n", + " (pl.col(\"d\") - pl.col(\"d\").min()) / (pl.col(\"d\").max() - pl.col(\"d\").min())\n", + " ).alias(\"d\"),\n", + " ]\n", + ")\n", "\n", "df" ] @@ -280,7 +217,7 @@ " \n", "```\n", "\n", - "So here it would be:" + "So here it would be:\n" ] }, { @@ -291,7 +228,7 @@ "```python\n", "def rescale(x):\n", " return (x - x.min()) / (x.max() - x.min())\n", - "```" + "```\n" ] }, { @@ -299,12 +236,12 @@ "id": "22f6afb7", "metadata": {}, "source": [ - "There is still some duplication in this code. We're computing the minimum of the data twice, so it makes sense to do it in once:" + "There is still some duplication in this code. We're computing the minimum of the data twice, so it makes sense to do it in once:\n" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "406648b6", "metadata": {}, "outputs": [], @@ -321,20 +258,19 @@ "source": [ "Pulling out intermediate calculations into named variables is a good practice because it makes it more clear what the code is doing.\n", "\n", - "\n", "There are three key steps to creating a new function:\n", "\n", - "1. You need to pick a *name* for the function. Here we've used `rescale` because this function rescales a vector to lie between 0 and 1.\n", + "1. You need to pick a _name_ for the function. Here we've used `rescale` because this function rescales a vector to lie between 0 and 1.\n", "\n", - "1. You list the inputs, or *arguments*, to the function inside `function`.\n", + "1. You list the inputs, or _arguments_, to the function inside `function`.\n", " Here we have just one argument.\n", - " If we had more the call would look like `function(x, y, z)`. (We might also have a named *keyword argument* such as `data=` following the *arguments*.)\n", + " If we had more the call would look like `function(x, y, z)`. (We might also have a named _keyword argument_ such as `data=` following the _arguments_.)\n", "\n", - "3. You place the code you have developed in the *body* of the function, a block that immediately follows `function(...):`.\n", + "1. You place the code you have developed in the _body_ of the function, a block that immediately follows `function(...):`.\n", "\n", "Note the overall process: we only made the function after we'd figured out how to make it work with a simple input. It's easier to start with working code and turn it into a function; it's harder to create a function and then try to make it work.\n", "\n", - "At this point it's a good idea to check your function with a few different inputs:" + "At this point it's a good idea to check your function with a few different inputs:\n" ] }, { @@ -365,22 +301,24 @@ "As you write more and more functions you'll eventually want to convert these informal, interactive tests into formal, automated tests.\n", "That process is called unit testing. Unfortunately, it's beyond the scope of this book.\n", "\n", - "We can simplify the original example now that we have a function:" + "We can simplify the original example now that we have a function:\n" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "ce066e4f", "metadata": {}, "outputs": [], "source": [ - "df = df.with_columns([\n", - " rescale(pl.col(\"a\")).alias(\"a\"),\n", - " rescale(pl.col(\"b\")).alias(\"b\"),\n", - " rescale(pl.col(\"c\")).alias(\"c\"),\n", - " rescale(pl.col(\"d\")).alias(\"d\")\n", - "])" + "df = df.with_columns(\n", + " [\n", + " rescale(pl.col(\"a\")).alias(\"a\"),\n", + " rescale(pl.col(\"b\")).alias(\"b\"),\n", + " rescale(pl.col(\"c\")).alias(\"c\"),\n", + " rescale(pl.col(\"d\")).alias(\"d\"),\n", + " ]\n", + ")" ] }, { @@ -391,7 +329,7 @@ "Compared to the original, this code is easier to understand and we've eliminated one class of copy-and-paste errors. There is still quite a bit of duplication since we're doing the same thing to multiple columns; we can actually remove this too, but we'll cover how to do that later in the book.\n", "\n", "Another advantage of functions is that if our requirements change, we only need to make the change in one place.\n", - "For example, we might discover that some of our variables include infinite values, and `rescale()` (effectively) fails:" + "For example, we might discover that some of our variables include infinite values, and `rescale()` (effectively) fails:\n" ] }, { @@ -401,7 +339,7 @@ "metadata": {}, "outputs": [], "source": [ - "rescale(pl.Series([1, 2, 3, float('inf'), 5], strict=False))" + "rescale(pl.Series([1, 2, 3, float(\"inf\"), 5], strict=False))" ] }, { @@ -409,7 +347,7 @@ "id": "f29a050e", "metadata": {}, "source": [ - "Because we've extracted the code into a function, we only need to make the fix in one place:" + "Because we've extracted the code into a function, we only need to make the fix in one place:\n" ] }, { @@ -420,11 +358,12 @@ "outputs": [], "source": [ "def rescale(x):\n", - " x = x.cast(pl.Float64).replace(float('inf'), None)\n", + " x = x.cast(pl.Float64).replace(float(\"inf\"), None)\n", " minimum = x.min()\n", " return (x - minimum) / (x.max() - minimum)\n", "\n", - "rescale(pl.Series([1, 2, 3, float('inf'), 5], strict=False))" + "\n", + "rescale(pl.Series([1, 2, 3, float(\"inf\"), 5], strict=False))" ] }, { @@ -433,7 +372,7 @@ "metadata": {}, "source": [ "This is an important part of the \"do not repeat yourself\" (or DRY) principle.\n", - "The more repetition you have in your code, the more places you need to remember to update when things change (and they always do!), and the more likely you are to create bugs over time." + "The more repetition you have in your code, the more places you need to remember to update when things change (and they always do!), and the more likely you are to create bugs over time.\n" ] }, { @@ -471,7 +410,7 @@ "text_input()\n", "```\n", "\n", - "A good example of this design is the **polars** package: if you don't remember exactly which function you need to read in data, you can type `pl.read_` and jog your memory as the autocomplete brings up the options." + "A good example of this design is the **polars** package: if you don't remember exactly which function you need to read in data, you can type `pl.read_` and jog your memory as the autocomplete brings up the options.\n" ] }, { @@ -495,10 +434,9 @@ "print(str_variable)\n", "```\n", "\n", - "This would raise an error, because as far as your general code is concerned `str_variable` doesn't exist outside of the function. This is an example of a *local* variable, one that only exists within a function.\n", - "\n", + "This would raise an error, because as far as your general code is concerned `str_variable` doesn't exist outside of the function. This is an example of a _local_ variable, one that only exists within a function.\n", "\n", - "If you want to create variables inside a function and have them persist, you need to explicitly pass them out using, for example `return str_variable` like this:" + "If you want to create variables inside a function and have them persist, you need to explicitly pass them out using, for example `return str_variable` like this:\n" ] }, { @@ -524,7 +462,7 @@ "source": [ "**Global**\n", "\n", - "A variable declared outside of a function is known as a global variable because it is accessible everywhere:" + "A variable declared outside of a function is known as a global variable because it is accessible everywhere:\n" ] }, { @@ -547,9 +485,6 @@ } ], "metadata": { - "interpreter": { - "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" - }, "jupytext": { "cell_metadata_filter": "-all", "encoding": "# -*- coding: utf-8 -*-", @@ -557,7 +492,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "python4ds", "language": "python", "name": "python3" }, From 05832e7febcd0c37e5a8cd6ca08e1ea790054198 Mon Sep 17 00:00:00 2001 From: Uchenna Ugoh Date: Sat, 25 Apr 2026 00:01:11 +0100 Subject: [PATCH 29/34] Updated workflow-styles --- workflow-style.ipynb | 85 +++++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/workflow-style.ipynb b/workflow-style.ipynb index ab2eb53..6921e86 100644 --- a/workflow-style.ipynb +++ b/workflow-style.ipynb @@ -9,11 +9,11 @@ "\n", "Good coding style is like correct punctuation: you can manage without it, butitsuremakesthingseasiertoread. Even as a very new programmer it's a good idea to work on your code style. Use a consistent style makes it easier for others (including future-you!) to read your work, and is particularly important if you need to get help from someone else.\n", "\n", - "This chapter will introduce you to some important style points drawn from [Clean Code in Python](https://testdriven.io/blog/clean-code-python/), [Tips for Better Coding](https://aeturrell.github.io/coding-for-economists/code-best-practice.html) from *Coding for Economists*, the UK Government Statistical Service’s [Quality Assurance of Code for Analysis and Research](https://best-practice-and-impact.github.io/qa-of-code-guidance/intro.html) guidance, and the bible of Python style guides, [PEP 8 — Style Guide for Python Code](https://peps.python.org/pep-0008/).\n", + "This chapter will introduce you to some important style points drawn from [Clean Code in Python](https://testdriven.io/blog/clean-code-python/), [Tips for Better Coding](https://aeturrell.github.io/coding-for-economists/code-best-practice.html) from _Coding for Economists_, the UK Government Statistical Service’s [Quality Assurance of Code for Analysis and Research](https://best-practice-and-impact.github.io/qa-of-code-guidance/intro.html) guidance, and the bible of Python style guides, [PEP 8 — Style Guide for Python Code](https://peps.python.org/pep-0008/).\n", "\n", "Styling your code will feel a bit tedious to start with, but if you practice it, it will soon become second nature. Additionally, there are some great tools to quickly restyle existing code, like the [Black](https://black.readthedocs.io/) Python package (\"you can have any colour you like, as long as it's black\").\n", "\n", - "Once you've installed Black by running `uv tool install black`, you can use it on the command line (aka the terminal) within Visual Studio Code. Open up a terminal by clicking 'Terminal -> New Terminal' and then run `black *.py` to apply a standard code style to all Python scripts in the current directory." + "Once you've installed Black by running `uv tool install black`, you can use it on the command line (aka the terminal) within Visual Studio Code. Open up a terminal by clicking 'Terminal -> New Terminal' and then run `black *.py` to apply a standard code style to all Python scripts in the current directory.\n" ] }, { @@ -23,44 +23,47 @@ "source": [ "## Names\n", "\n", - "First, names matter. Use meaningful names for variables, functions, or whatever it is you're naming. Avoid abbreviations that you understand *now* but which will be unclear to others, or future you. For example, use `real_wage_hourly` over `re_wg_ph`. I know it's tempting to use `temp` but you'll feel silly later when you can't for the life of you remember what `temp` does or is. A good trick when naming booleans (variables that are either true or false) is to use `is` followed by what the boolean variable refers to, for example `is_married`.\n", + "First, names matter. Use meaningful names for variables, functions, or whatever it is you're naming. Avoid abbreviations that you understand _now_ but which will be unclear to others, or future you. For example, use `real_wage_hourly` over `re_wg_ph`. I know it's tempting to use `temp` but you'll feel silly later when you can't for the life of you remember what `temp` does or is. A good trick when naming booleans (variables that are either true or false) is to use `is` followed by what the boolean variable refers to, for example `is_married`.\n", "\n", "As well as this general tip, Python has conventions on naming different kinds of variables. The naming convention for almost all objects is lower case separated by underscores, e.g. `a_variable=10` or ‘this_is_a_script.py’. This style of naming is also known as snake case. There are different naming conventions though—[Allison Horst](https://twitter.com/allison_horst) made this fantastic cartoon of the different conventions that are in use.\n", "\n", - "![Different naming conventions. Artwork by \\@allison_horst.](https://github.com/aeturrell/coding-for-economists/raw/main/img/in_that_case.jpg) Different naming conventions. Artwork by \\@allison_horst.\n", + "![Different naming conventions. Artwork by @allison_horst.](https://github.com/aeturrell/coding-for-economists/raw/main/img/in_that_case.jpg) Different naming conventions. Artwork by \\@allison_horst.\n", "\n", "There are three exceptions to the snake case convention: classes, which should be in camel case, eg `ThisIsAClass`; constants, which are in capital snake case, eg `THIS_IS_A_CONSTANT`; and packages, which are typically without spaces or underscores and are lowercase `thisisapackage`.\n", "\n", - "For some quick shortcuts to re-naming columns in **pandas** data frames or other string variables, try the unicode-friendly [**slugify**](https://github.com/un33k/python-slugify) library or the `clean_headers()` function from the [**dataprep**](https://docs.dataprep.ai/index.html) library.\n", + "For some quick shortcuts to re-naming columns in **polars** data frames or other string variables, try the unicode-friendly [**slugify**](https://github.com/un33k/python-slugify) library or the `clean_headers()` function from the [**dataprep**](https://docs.dataprep.ai/index.html) library.\n", "\n", "The better named your variables, the clearer your code will be--and the fewer comments you will need to write!\n", "\n", "In summary:\n", + "\n", "- use descriptive variable names that reveal your intention, eg `days_since_treatment`\n", "- avoid using ambiguous abbreviations in names, eg use `real_wage_hourly` over `rw_ph`\n", "- always use the same vocabulary, eg don't switch from `worker_type` to `employee_type`\n", "- avoid 'magic numbers', eg numbers in your code that set a key parameter. Set these as named constants instead. Here's an example:\n", - " ```python\n", - " import random\n", "\n", - " # This is bad\n", - " def roll():\n", - " return random.randint(0, 36) # magic number!\n", + " ```python\n", + " import random\n", + "\n", + " # This is bad\n", + " def roll():\n", + " return random.randint(0, 36) # magic number!\n", "\n", - " # This is good\n", - " MAX_INT_VALUE = 36\n", + " # This is good\n", + " MAX_INT_VALUE = 36\n", + "\n", + " def roll():\n", + " return random.randint(0, MAX_INT_VALUE)\n", + " ```\n", "\n", - " def roll():\n", - " return random.randint(0, MAX_INT_VALUE)\n", - " ```\n", "- use verbs for function names, eg `get_regression()`\n", "- use consistent verbs for function names, don't use `get_score()` and `grab_results()` (instead use `get` for both)\n", "- variable names should be snake_case and all lowercase, eg `first_name`\n", - " - class names should be CamelCase, eg `MyClass`\n", - " - function names should be snake_case and all lowercase, eg `quick_sort()`\n", - " - constants should be snake_case and all uppercase, eg `PI = 3.14159`\n", - " - modules should have short, snake_case names and all lowercase, eg `pandas`\n", - " - single quotes and double quotes are equivalent so pick one and be consistent—most automatic formatters prefer `\"`" + "- class names should be CamelCase, eg `MyClass`\n", + "- function names should be snake_case and all lowercase, eg `quick_sort()`\n", + "- constants should be snake_case and all uppercase, eg `PI = 3.14159`\n", + "- modules should have short, snake_case names and all lowercase, eg `pandas`\n", + "- single quotes and double quotes are equivalent so pick one and be consistent—most automatic formatters prefer `\"`\n" ] }, { @@ -88,7 +91,7 @@ "list_var = [1,2,3,4]\n", "# or\n", "list_var = [1 , 2 , 3 , 4]\n", - "```" + "```\n" ] }, { @@ -98,9 +101,9 @@ "source": [ "## Code Comments\n", "\n", - "As mentioned previously, Python will ignore any text after `#`. This allows to you to write **comments**, text that is ignored by Python but can be read by other humans. Comments can be helpful for briefly describing what the subsequent code does: use them to provide extra contextual information that *isn't* conveyed by function and variable names.\n", + "As mentioned previously, Python will ignore any text after `#`. This allows to you to write **comments**, text that is ignored by Python but can be read by other humans. Comments can be helpful for briefly describing what the subsequent code does: use them to provide extra contextual information that _isn't_ conveyed by function and variable names.\n", "\n", - "Actually, well-written code needs *fewer* comments because it's more evidence what's going on. And it's tempting not to update comments even when code changes. So do comment, but see if you can make the code tell its own story first.\n", + "Actually, well-written code needs _fewer_ comments because it's more evidence what's going on. And it's tempting not to update comments even when code changes. So do comment, but see if you can make the code tell its own story first.\n", "\n", "Also, avoid \"noise\" comments that tell you what you already know from just looking at the code.\n", "\n", @@ -109,17 +112,16 @@ "Finally, functions come with their own special type of comments called a doc string. Here's an example that tells us all about the functions inputs and outputs, including the type of input and output (here a data frame, also known as `pd.DataFrame`).\n", "\n", "```python\n", - "def round_dataframe(df: pd.DataFrame) -> pd.DataFrame:\n", - " \"\"\"Rounds numeric columns in dataframe to 2 s.f.\n", + "def get_numeric_mean(df: pl.DataFrame) -> pl.DataFrame:\n", + " \"\"\"Returns the mean of each numeric column in the DataFrame.\n", " Args:\n", - " df (pd.DataFrame): Input dataframe\n", + " df (pl.DataFrame): Input dataframe\n", " Returns:\n", - " pd.DataFrame: Dataframe with numbers rounded to 2 s.f.\n", + " pl.DataFrame: Dataframe of mean value of each numeric column.\n", " \"\"\"\n", - " for col in df.select_dtypes(\"number\"):\n", - " df[col] = df[col].apply(lambda x: float(f'{float(f\"{x:.2g}\"):g}'))\n", - " return df\n", - "```" + " df_mean = df.select(S.numeric()).mean()\n", + " return df_mean\n", + "```\n" ] }, { @@ -141,7 +143,7 @@ " return result\n", "```\n", "\n", - "When using *method chaining* (something you can see in action in [](data-transform)) it's necessary to put the chain inside parentheses and it's good practice to use a new line for every method. The code snippet below gives an example of what good looks like:" + "When using _method chaining_ (something you can see in action in [](data-transform)) it's necessary to put the chain inside parentheses and it's good practice to use a new line for every method. The code snippet below gives an example of what good looks like:\n" ] }, { @@ -151,9 +153,9 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import polars as pl\n", "\n", - "df = pd.DataFrame(\n", + "df = pl.DataFrame(\n", " data={\n", " \"col0\": [0, 0, 0, 0],\n", " \"col1\": [0, 0, 0, 0],\n", @@ -161,13 +163,14 @@ " \"col3\": [\"a\", \"b\", \"b\", \"a\"],\n", " \"col4\": [\"alpha\", \"gamma\", \"gamma\", \"gamma\"],\n", " },\n", - " index=[\"row\" + str(i) for i in range(4)],\n", ")\n", "\n", "\n", "# Chaining inside parentheses works\n", "\n", - "results = df.groupby([\"col3\", \"col4\"]).agg({\"col1\": \"count\", \"col2\": \"mean\"})\n", + "results = df.group_by([\"col3\", \"col4\"]).agg(\n", + " pl.col(\"col1\").count(), pl.col(\"col2\").mean()\n", + ")\n", "\n", "results" ] @@ -177,13 +180,13 @@ "id": "1d6f3bf8", "metadata": {}, "source": [ - "And this is what *not* to do:\n", + "And this is what _not_ to do:\n", "\n", "```python\n", "results = df\n", - " .groupby([\"col3\", \"col4\"]).agg({\"col1\": \"count\", \"col2\": \"mean\"})\n", + " .group_by([\"col3\", \"col4\"]).agg(pl.col(\"col1\").count(), pl.col(\"col2\").mean())\n", "\n", - "```" + "```\n" ] }, { @@ -193,7 +196,7 @@ "source": [ "## Principles of Clean Code\n", "\n", - "While automation can help apply style, it can't help you write *clean code*. Clean code is a set of rules and principles that helps to keep your code readable, maintainable, and extendable. Writing code is easy; writing clean code is hard! However, if you follow these principles, you won't go far wong.\n", + "While automation can help apply style, it can't help you write _clean code_. Clean code is a set of rules and principles that helps to keep your code readable, maintainable, and extendable. Writing code is easy; writing clean code is hard! However, if you follow these principles, you won't go far wong.\n", "\n", "### Do not repeat yourself (DRY)\n", "\n", @@ -239,7 +242,7 @@ "\n", "def lowercase(text):\n", " return text.lower()\n", - "```" + "```\n" ] } ], From ea297ed9a467e997d6c2aee360a4fb8c01943de1 Mon Sep 17 00:00:00 2001 From: Nwabueze Ugoh Date: Tue, 28 Apr 2026 13:44:18 +0100 Subject: [PATCH 30/34] Update notebooks to enhance execution counts and output displays - Added execution counts to various code cells across multiple notebooks for better tracking of code execution. - Enhanced output displays in `boolean-data.ipynb`, `categorical-data.ipynb`, `communicate-plots.ipynb`, `data-import.ipynb`, `data-tidy.ipynb`, `data-transform.ipynb`, and `data-visualise.ipynb` to provide clearer results and visualizations. - Improved overall consistency in notebook formatting and presentation. --- boolean-data.ipynb | 1135 +- categorical-data.ipynb | 359 +- communicate-plots.ipynb | 7792 ++++++++++++- data-import.ipynb | 326 +- data-tidy.ipynb | 1013 +- data-transform.ipynb | 1486 ++- data-visualise.ipynb | 2522 ++++- data/bake_sale.xlsx | Bin 6218 -> 6192 bytes databases.ipynb | 420 +- dates-and-times.ipynb | 11631 ++++++++++++++++++- exploratory-data-analysis.ipynb | 17753 +++++++++++++++++++++++++++++- functions.ipynb | 249 +- introduction.ipynb | 122 +- iteration.ipynb | 549 +- joins.ipynb | 374 +- missing-values.ipynb | 1470 ++- numbers.ipynb | 7333 +++++++++++- prerequisites.ipynb | 2 +- rectangling.ipynb | 359 +- regex.ipynb | 105 +- spreadsheets.ipynb | 418 +- strings.ipynb | 810 +- vis-layers.ipynb | 3902 ++++++- webscraping-and-apis.ipynb | 500 +- whole-game.ipynb | 110 +- workflow-basics.ipynb | 180 +- workflow-style.ipynb | 36 +- workflow-writing-code.ipynb | 2 +- 28 files changed, 59878 insertions(+), 1080 deletions(-) diff --git a/boolean-data.ipynb b/boolean-data.ipynb index 931bbb2..7da9c0b 100644 --- a/boolean-data.ipynb +++ b/boolean-data.ipynb @@ -22,10 +22,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "7e35b9fc", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "bool_variable = True\n", "bool_variable" @@ -51,10 +62,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "590cd75d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "not True" ] @@ -86,10 +108,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "51622575", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "False\n" + ] + } + ], "source": [ "boolean_condition = 10 == 20\n", "print(boolean_condition)" @@ -115,10 +145,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "0c550daa", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ada, you achieved a high score.\n", + "You could be called Smith or have a high score\n", + "You are not called Smith and you have a high score\n" + ] + } + ], "source": [ "name = \"Ada\"\n", "score = 99\n", @@ -143,10 +183,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "7420e1c1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "False\n" + ] + } + ], "source": [ "name_list = [\"Ada\", \"Adam\"]\n", "name_list_two = [\"Ada\", \"Adam\"]\n", @@ -176,10 +225,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "39caa7be", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "False\n" + ] + } + ], "source": [ "name_list = [\"Lovelace\", \"Smith\", \"Hopper\", \"Babbage\"]\n", "\n", @@ -210,10 +268,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "95794e71", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "High score!\n" + ] + } + ], "source": [ "score = 98\n", "\n", @@ -243,10 +309,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "cd1cd061", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "a, b = 3, 6\n", "\n", @@ -265,10 +342,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "59638407", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "[x for x in range(12)]" ] @@ -283,10 +371,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "8e8072ea", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[0, 2, 4, 6, 8, 10]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "[x for x in range(12) if x % 2 == 0]" ] @@ -301,10 +400,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "ec01f460", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[0,\n", + " 'Not divisible by 2',\n", + " 2,\n", + " 'Not divisible by 2',\n", + " 4,\n", + " 'Not divisible by 2',\n", + " 6,\n", + " 'Not divisible by 2',\n", + " 8,\n", + " 'Not divisible by 2',\n", + " 10,\n", + " 'Not divisible by 2']" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "[x if x % 2 == 0 else \"Not divisible by 2\" for x in range(12)]" ] @@ -328,10 +449,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "dc605a93", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Falsy\n" + ] + } + ], "source": [ "listy = []\n", "other_listy = [1, 2, 3]\n", @@ -344,10 +473,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "da8fe682", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Truthy\n" + ] + } + ], "source": [ "if not (other_listy):\n", " print(\"Falsy\")\n", @@ -365,10 +502,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "d80ba0be", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Falsy\n" + ] + } + ], "source": [ "if not 0:\n", " print(\"Falsy\")\n", @@ -378,10 +523,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "1973d44d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Truthy\n" + ] + } + ], "source": [ "if not [0, 0, 0]:\n", " print(\"Falsy\")\n", @@ -399,10 +552,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "62840c4a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Falsy\n" + ] + } + ], "source": [ "if not None:\n", " print(\"Falsy\")\n", @@ -432,10 +593,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "bdcb09a5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "any([True, False, False])" ] @@ -450,10 +622,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "2f666185", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "all([True, True, True, True])" ] @@ -468,10 +651,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "78777d9c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "all([0, 0, 0, 1])" ] @@ -490,10 +684,79 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "7f338fd7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
bool_col_1bool_col_2
0FalseTrue
1FalseFalse
2FalseTrue
3TrueFalse
4TrueTrue
\n", + "
" + ], + "text/plain": [ + " bool_col_1 bool_col_2\n", + "0 False True\n", + "1 False False\n", + "2 False True\n", + "3 True False\n", + "4 True True" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import pandas as pd\n", "\n", @@ -516,10 +779,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "9cdaec7a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 True\n", + "1 False\n", + "2 True\n", + "3 True\n", + "4 True\n", + "dtype: bool" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[\"bool_col_1\"] | df[\"bool_col_2\"]" ] @@ -534,10 +813,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "89ee3e44", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "bool_col_1 2\n", + "bool_col_2 3\n", + "dtype: int64" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.sum()" ] @@ -552,10 +844,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "5e30cee7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 False\n", + "1 True\n", + "2 False\n", + "3 True\n", + "4 True\n", + "Name: bool_col, dtype: bool" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = pd.DataFrame.from_dict({\"bool_col\": [0, 1, 0, 1, 1]})\n", "df[\"bool_col\"].astype(bool)" @@ -574,10 +882,127 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "9f63005f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
caratcutcolorclaritydepthtablepricexyz
00.23IdealESI261.555.03263.953.982.43
10.21PremiumESI159.861.03263.893.842.31
20.23GoodEVS156.965.03274.054.072.31
30.29PremiumIVS262.458.03344.204.232.63
40.31GoodJSI263.358.03354.344.352.75
\n", + "
" + ], + "text/plain": [ + " carat cut color clarity depth table price x y z\n", + "0 0.23 Ideal E SI2 61.5 55.0 326 3.95 3.98 2.43\n", + "1 0.21 Premium E SI1 59.8 61.0 326 3.89 3.84 2.31\n", + "2 0.23 Good E VS1 56.9 65.0 327 4.05 4.07 2.31\n", + "3 0.29 Premium I VS2 62.4 58.0 334 4.20 4.23 2.63\n", + "4 0.31 Good J SI2 63.3 58.0 335 4.34 4.35 2.75" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "diamonds = pd.read_csv(\n", " \"https://github.com/mwaskom/seaborn-data/raw/master/diamonds.csv\"\n", @@ -595,10 +1020,220 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "id": "7a27f0a0", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
caratcutcolorclaritydepthtablepricexyzexpensive
93380.94PremiumESI160.058.045806.446.373.84True
11800.71IdealGVS162.757.029305.695.733.58True
283340.33PremiumGVS161.358.06664.424.462.72False
139191.23Very GoodJVVS261.257.056656.866.924.22True
264742.37IdealJVS262.257.0160598.528.585.32True
122331.19FairISI164.958.051986.646.554.28True
59570.90Very GoodFSI162.660.039506.106.143.83True
483250.33GoodFSI161.862.05364.404.452.74False
257421.51IdealGVS161.157.0146747.417.384.52True
434010.59Very GoodFSI161.757.014125.395.433.34True
\n", + "
" + ], + "text/plain": [ + " carat cut color clarity depth table price x y z \\\n", + "9338 0.94 Premium E SI1 60.0 58.0 4580 6.44 6.37 3.84 \n", + "1180 0.71 Ideal G VS1 62.7 57.0 2930 5.69 5.73 3.58 \n", + "28334 0.33 Premium G VS1 61.3 58.0 666 4.42 4.46 2.72 \n", + "13919 1.23 Very Good J VVS2 61.2 57.0 5665 6.86 6.92 4.22 \n", + "26474 2.37 Ideal J VS2 62.2 57.0 16059 8.52 8.58 5.32 \n", + "12233 1.19 Fair I SI1 64.9 58.0 5198 6.64 6.55 4.28 \n", + "5957 0.90 Very Good F SI1 62.6 60.0 3950 6.10 6.14 3.83 \n", + "48325 0.33 Good F SI1 61.8 62.0 536 4.40 4.45 2.74 \n", + "25742 1.51 Ideal G VS1 61.1 57.0 14674 7.41 7.38 4.52 \n", + "43401 0.59 Very Good F SI1 61.7 57.0 1412 5.39 5.43 3.34 \n", + "\n", + " expensive \n", + "9338 True \n", + "1180 True \n", + "28334 False \n", + "13919 True \n", + "26474 True \n", + "12233 True \n", + "5957 True \n", + "48325 False \n", + "25742 True \n", + "43401 True " + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "diamonds[\"expensive\"] = diamonds[\"price\"] > 1000\n", "diamonds.sample(10)" @@ -614,10 +1249,140 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "id": "c78a6d47", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
caratcutcolorclaritydepthtablepricexyzexpensive
00.23IdealESI261.555.03263.953.982.43False
10.21PremiumESI159.861.03263.893.842.31False
20.23GoodEVS156.965.03274.054.072.31False
30.29PremiumIVS262.458.03344.204.232.63False
40.31GoodJSI263.358.03354.344.352.75False
\n", + "
" + ], + "text/plain": [ + " carat cut color clarity depth table price x y z \\\n", + "0 0.23 Ideal E SI2 61.5 55.0 326 3.95 3.98 2.43 \n", + "1 0.21 Premium E SI1 59.8 61.0 326 3.89 3.84 2.31 \n", + "2 0.23 Good E VS1 56.9 65.0 327 4.05 4.07 2.31 \n", + "3 0.29 Premium I VS2 62.4 58.0 334 4.20 4.23 2.63 \n", + "4 0.31 Good J SI2 63.3 58.0 335 4.34 4.35 2.75 \n", + "\n", + " expensive \n", + "0 False \n", + "1 False \n", + "2 False \n", + "3 False \n", + "4 False " + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "diamonds.assign(expensive=lambda x: x[\"price\"] > 1000).head()" ] @@ -632,10 +1397,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "d12f537d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "array([False, False, False, False, False, False, False, True, True,\n", + " True, False])" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "diamonds.columns.isin([\"x\", \"y\", \"z\"])" ] @@ -652,10 +1429,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "35e73305", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "np.True_" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "diamonds[\"expensive\"].any()" ] @@ -672,10 +1460,239 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "id": "d1bbb0fa", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
caratcutcolorclaritydepthtablepricexyzexpensive
10.21PremiumESI159.861.03263.893.842.31False
80.22FairEVS265.161.03373.873.782.49False
110.23IdealJVS162.856.03403.933.902.46False
120.22PremiumFSI160.461.03423.883.842.33False
140.20PremiumESI260.262.03453.793.752.27False
....................................
539280.79PremiumESI261.458.027566.035.963.68True
539290.71IdealGVS161.456.027565.765.733.53True
539300.71PremiumESI160.555.027565.795.743.49True
539310.71PremiumFSI159.862.027565.745.733.43True
539380.86PremiumHSI261.058.027576.156.123.74True
\n", + "

23423 rows × 11 columns

\n", + "
" + ], + "text/plain": [ + " carat cut color clarity depth table price x y z \\\n", + "1 0.21 Premium E SI1 59.8 61.0 326 3.89 3.84 2.31 \n", + "8 0.22 Fair E VS2 65.1 61.0 337 3.87 3.78 2.49 \n", + "11 0.23 Ideal J VS1 62.8 56.0 340 3.93 3.90 2.46 \n", + "12 0.22 Premium F SI1 60.4 61.0 342 3.88 3.84 2.33 \n", + "14 0.20 Premium E SI2 60.2 62.0 345 3.79 3.75 2.27 \n", + "... ... ... ... ... ... ... ... ... ... ... \n", + "53928 0.79 Premium E SI2 61.4 58.0 2756 6.03 5.96 3.68 \n", + "53929 0.71 Ideal G VS1 61.4 56.0 2756 5.76 5.73 3.53 \n", + "53930 0.71 Premium E SI1 60.5 55.0 2756 5.79 5.74 3.49 \n", + "53931 0.71 Premium F SI1 59.8 62.0 2756 5.74 5.73 3.43 \n", + "53938 0.86 Premium H SI2 61.0 58.0 2757 6.15 6.12 3.74 \n", + "\n", + " expensive \n", + "1 False \n", + "8 False \n", + "11 False \n", + "12 False \n", + "14 False \n", + "... ... \n", + "53928 True \n", + "53929 True \n", + "53930 True \n", + "53931 True \n", + "53938 True \n", + "\n", + "[23423 rows x 11 columns]" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "diamonds[diamonds[\"x\"] > diamonds[\"y\"]]" ] @@ -714,7 +1731,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/categorical-data.ipynb b/categorical-data.ipynb index 5796a26..6cc6776 100644 --- a/categorical-data.ipynb +++ b/categorical-data.ipynb @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "51a55374", "metadata": { "tags": [ @@ -65,10 +65,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "535ef959", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 a\n", + "1 b\n", + "2 c\n", + "3 a\n", + "Name: A, dtype: category\n", + "Categories (3, object): ['a', 'b', 'c']" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import numpy as np\n", "import pandas as pd\n", @@ -91,10 +107,79 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "358c83bb", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
valuegroup
03630 - 39
16160 - 69
28980 - 89
32820 - 29
48180 - 89
\n", + "
" + ], + "text/plain": [ + " value group\n", + "0 36 30 - 39\n", + "1 61 60 - 69\n", + "2 89 80 - 89\n", + "3 28 20 - 29\n", + "4 81 80 - 89" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = pd.DataFrame({\"value\": np.random.randint(0, 100, 20)})\n", "labels = [f\"{i} - {i+9}\" for i in range(0, 100, 10)]\n", @@ -114,10 +199,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "fb389105", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[NaN, 'b', 'c', NaN, 'd', NaN, 'c']\n", + "Categories (3, object): ['b', 'c', 'd']" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "raw_cat = pd.Categorical(\n", " [\"a\", \"b\", \"c\", \"a\", \"d\", \"a\", \"c\"], categories=[\"b\", \"c\", \"d\"]\n", @@ -135,10 +232,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "0497fc16", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 NaN\n", + "1 b\n", + "2 c\n", + "3 NaN\n", + "4 d\n", + "5 NaN\n", + "6 c\n", + "Name: cat_type, dtype: category\n", + "Categories (3, object): ['b', 'c', 'd']" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = pd.DataFrame(raw_cat, columns=[\"cat_type\"])\n", "df[\"cat_type\"]" @@ -162,10 +278,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "f7520d3d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['a', 'b', 'c', 'a', 'd', 'a', 'c']\n", + "Categories (4, object): ['a' < 'b' < 'c' < 'd']" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ordered_cat = pd.Categorical(\n", " [\"a\", \"b\", \"c\", \"a\", \"d\", \"a\", \"c\"],\n", @@ -189,20 +317,42 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "2caba354", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Index(['b', 'c', 'd'], dtype='object')" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[\"cat_type\"].cat.categories" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "5f1fe093", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[\"cat_type\"].cat.ordered" ] @@ -227,7 +377,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "097171b8", "metadata": {}, "outputs": [], @@ -245,10 +395,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "4ae6df38", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 NaN\n", + "1 alpha\n", + "2 beta\n", + "3 NaN\n", + "4 gamma\n", + "5 NaN\n", + "6 beta\n", + "Name: cat_type, dtype: category\n", + "Categories (4, object): ['alpha', 'beta', 'gamma', 'delta']" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[\"cat_type\"] = df[\"cat_type\"].cat.add_categories([\"delta\"])\n", "df[\"cat_type\"]" @@ -274,10 +443,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "19f5bdda", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "cat_type\n", + "beta 2\n", + "alpha 1\n", + "gamma 1\n", + "delta 0\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[\"cat_type\"].value_counts()" ] @@ -294,10 +479,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "f52d5d0d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 beta\n", + "Name: cat_type, dtype: category\n", + "Categories (4, object): ['alpha', 'beta', 'gamma', 'delta']" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[\"cat_type\"].mode()" ] @@ -312,10 +510,81 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "4d43e94d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60895/379284818.py:2: FutureWarning: 'M' is deprecated and will be removed in a future version, please use 'ME' instead.\n", + " pd.Series(pd.date_range(\"2015/05/01\", periods=5, freq=\"M\"), dtype=\"category\"),\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
datetime
02015-05-31
12015-06-30
22015-07-31
32015-08-31
42015-09-30
\n", + "
" + ], + "text/plain": [ + " datetime\n", + "0 2015-05-31\n", + "1 2015-06-30\n", + "2 2015-07-31\n", + "3 2015-08-31\n", + "4 2015-09-30" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "time_df = pd.DataFrame(\n", " pd.Series(pd.date_range(\"2015/05/01\", periods=5, freq=\"M\"), dtype=\"category\"),\n", @@ -326,10 +595,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "db697f86", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 5\n", + "1 6\n", + "2 7\n", + "3 8\n", + "4 9\n", + "Name: datetime, dtype: int32" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "time_df[\"datetime\"].dt.month" ] @@ -344,10 +629,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "13e7bc66", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 0\n", + "1 1\n", + "2 2\n", + "3 3\n", + "4 4\n", + "dtype: int8" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "time_df[\"datetime\"].cat.codes" ] @@ -378,7 +679,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/communicate-plots.ipynb b/communicate-plots.ipynb index fc14db8..f833d44 100644 --- a/communicate-plots.ipynb +++ b/communicate-plots.ipynb @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "78eeea41", "metadata": {}, "outputs": [], @@ -42,10 +42,51 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "ae4a818a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "import numpy as np\n", "import polars as pl\n", @@ -66,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "c36b4cd5", "metadata": {}, "outputs": [], @@ -88,10 +129,132 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "c7574bc6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point())" ] @@ -116,10 +279,162 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "24b3513e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -147,10 +462,165 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "6489a6bf", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -182,14 +652,153 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "683d547c", "metadata": { "tags": [ "remove-cell" ] }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"cty\", y=\"hwy\", color=\"drv\", shape=\"drv\"))\n", @@ -231,10 +840,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "60826a32", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (3, 4)
drvhwydispldrive_type
strf64f64str
"f"28.162.56"front-wheel drive"
"r"21.05.18"rear-wheel drive"
"4"19.174.0"4-wheel drive"
" + ], + "text/plain": [ + "shape: (3, 4)\n", + "┌─────┬───────┬───────┬───────────────────┐\n", + "│ drv ┆ hwy ┆ displ ┆ drive_type │\n", + "│ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ f64 ┆ f64 ┆ str │\n", + "╞═════╪═══════╪═══════╪═══════════════════╡\n", + "│ f ┆ 28.16 ┆ 2.56 ┆ front-wheel drive │\n", + "│ r ┆ 21.0 ┆ 5.18 ┆ rear-wheel drive │\n", + "│ 4 ┆ 19.17 ┆ 4.0 ┆ 4-wheel drive │\n", + "└─────┴───────┴───────┴───────────────────┘" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "mapping = {\n", " \"4\": \"4-wheel drive\",\n", @@ -263,10 +902,183 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "6f90c2aa", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\", color=\"drv\"))\n", @@ -296,10 +1108,254 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "bdcd79bb", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "potential_outliers = mpg.filter(\n", " (pl.col(\"hwy\") > 40) | ((pl.col(\"hwy\") > 20) & (pl.col(\"displ\") > 5))\n", @@ -336,10 +1392,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "d1e2cc3a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'Larger engine sizes tend to\\nhave lower fuel economy.'" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import textwrap\n", "\n", @@ -350,10 +1417,162 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "e8c09f57", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -485,10 +1704,137 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "a95604d8", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\", color=\"drv\"))\n", @@ -509,10 +1855,138 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "1a852304", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\", color=\"drv\"))\n", @@ -539,7 +2013,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "40ac230e", "metadata": {}, "outputs": [], @@ -559,10 +2033,158 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "1520bb3c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(diamonds, aes(x=\"cut\", y=\"price\"))\n", @@ -582,10 +2204,42 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "9d1f993a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (5, 6)
rownamesnamestartendpartyid
i64stri32i32stri64
1"Eisenhower"19531961"Republican"34
2"Kennedy"19611963"Democratic"35
3"Johnson"19631969"Democratic"36
4"Nixon"19691974"Republican"37
5"Ford"19741977"Republican"38
" + ], + "text/plain": [ + "shape: (5, 6)\n", + "┌──────────┬────────────┬───────┬──────┬────────────┬─────┐\n", + "│ rownames ┆ name ┆ start ┆ end ┆ party ┆ id │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ str ┆ i32 ┆ i32 ┆ str ┆ i64 │\n", + "╞══════════╪════════════╪═══════╪══════╪════════════╪═════╡\n", + "│ 1 ┆ Eisenhower ┆ 1953 ┆ 1961 ┆ Republican ┆ 34 │\n", + "│ 2 ┆ Kennedy ┆ 1961 ┆ 1963 ┆ Democratic ┆ 35 │\n", + "│ 3 ┆ Johnson ┆ 1963 ┆ 1969 ┆ Democratic ┆ 36 │\n", + "│ 4 ┆ Nixon ┆ 1969 ┆ 1974 ┆ Republican ┆ 37 │\n", + "│ 5 ┆ Ford ┆ 1974 ┆ 1977 ┆ Republican ┆ 38 │\n", + "└──────────┴────────────┴───────┴──────┴────────────┴─────┘" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "presidential = pl.read_csv(\n", " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/presidential.csv\",\n", @@ -601,10 +2255,128 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "7d88976d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(presidential, aes(x=\"start\", y=\"id\"))\n", @@ -631,10 +2403,355 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "52d6e86a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "base = ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point(aes(color=\"class\"))\n", "\n", @@ -673,10 +2790,130 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "2c1d3f8d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(\n", @@ -697,10 +2934,136 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "39b4ef8d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(diamonds, aes(x=\"carat\", y=\"price\"))\n", @@ -720,20 +3083,272 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "f06d7e40", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point(aes(color=\"drv\")))" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "6186b520", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -762,14 +3377,1235 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "bd347524", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:34:54.100390\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# | echo: false\n", "cmaps = [\n", @@ -871,14 +4707,1126 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "id": "d6350c71", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:34:54.622782\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# remove input\n", "for cmap_category, cmap_list in cmaps[3:4]:\n", @@ -887,14 +5835,1003 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "id": "0063a574", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:34:54.717286\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# remove input\n", "for cmap_category, cmap_list in cmaps[2:3]:\n", @@ -911,10 +6848,138 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "9751058d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "mini_presid = presidential.slice(5)\n", "\n", @@ -942,10 +7007,115 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "644fd814", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "prng = np.random.default_rng(1837) # prng=probabilistic random number generator\n", "df_rnd = pl.DataFrame(prng.standard_normal((1000, 2)), schema=[\"x\", \"y\"])\n", @@ -979,10 +7149,147 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "id": "25a29f38", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -993,10 +7300,147 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "id": "42318a59", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "mpg_condition = (\n", " (mpg[\"displ\"] >= 5) & (mpg[\"displ\"] <= 6) & (mpg[\"hwy\"] >= 10) & (mpg[\"hwy\"] <= 25)\n", @@ -1021,10 +7465,153 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "id": "03001d5e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -1037,10 +7624,153 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "id": "dc3bb833", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -1061,10 +7791,134 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "id": "aee538a8", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "suv = mpg.filter(mpg[\"class\"] == \"suv\")\n", "compact = mpg.filter(mpg[\"class\"] == \"compact\")\n", @@ -1073,10 +7927,134 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "id": "a82c8c23", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(compact, aes(x=\"displ\", y=\"hwy\", color=\"drv\")) + geom_point())" ] @@ -1091,7 +8069,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "id": "db6fce43", "metadata": {}, "outputs": [], @@ -1103,10 +8081,73 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "id": "dd9e6606", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(suv, aes(x=\"displ\", y=\"hwy\", color=\"drv\"))\n", @@ -1119,10 +8160,73 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "id": "bdd8b2c5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(compact, aes(x=\"displ\", y=\"hwy\", color=\"drv\"))\n", @@ -1172,10 +8276,148 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "id": "0b2364ca", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -1197,10 +8439,159 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "id": "67bfa9c8", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", color=\"drv\"))\n", @@ -1241,10 +8632,228 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "id": "a8081df4", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "p1 = ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point() + labs(title=\"Plot 1\")\n", "p2 = ggplot(mpg, aes(x=\"drv\", y=\"hwy\")) + geom_boxplot() + labs(title=\"Plot 2\")\n", @@ -1265,10 +8874,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "id": "710a6a4f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'/Users/omagic/Documents/GitHub/python4DSpolars/chart.svg'" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ggsave(p1, \"chart.svg\", path=\".\")" ] @@ -1283,17 +8903,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "id": "bc831b1b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ls: stdout: Broken pipe\r\n" + ] + } + ], "source": [ "!ls | grep *.svg" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "id": "9cc10ab7", "metadata": { "tags": [ @@ -1346,7 +8974,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/data-import.ipynb b/data-import.ipynb index c240a16..9369dde 100644 --- a/data-import.ipynb +++ b/data-import.ipynb @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "1cf01bda", "metadata": {}, "outputs": [], @@ -54,10 +54,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "eca85c47", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Student ID,Full Name,favourite.food,mealPlan,AGE\r", + "\r\n", + "1,Sunil Huffmann,Strawberry yoghurt,Lunch only,4\r", + "\r\n", + "2,Barclay Lynn,French fries,Lunch only,5\r", + "\r\n", + "3,Jayendra Lyne,N/A,Breakfast and lunch,7\r", + "\r\n", + "4,Leon Rossini,Anchovies,Lunch only,8\r", + "\r\n", + "5,Chidiegwu Dunkel,Pizza,Breakfast and lunch,five\r", + "\r\n", + "6,Güvenç Attila,Ice cream,Lunch only,6" + ] + } + ], "source": [ "! cat data/students.csv" ] @@ -72,10 +92,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "232fdfef", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 5)
Student IDFull Namefavourite.foodmealPlanAGE
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only""8"
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""five"
6"Güvenç Attila""Ice cream""Lunch only""6"
" + ], + "text/plain": [ + "shape: (6, 5)\n", + "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", + "│ Student ID ┆ Full Name ┆ favourite.food ┆ mealPlan ┆ AGE │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", + "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", + "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", + "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", + "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", + "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ 8 │\n", + "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ five │\n", + "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", + "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "students = pl.read_csv(\"data/students.csv\")\n", "students" @@ -133,10 +186,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "51969364", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 5)
student_idfull_namefavourite_foodmeal_planage
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only""8"
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""five"
6"Güvenç Attila""Ice cream""Lunch only""6"
" + ], + "text/plain": [ + "shape: (6, 5)\n", + "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", + "│ student_id ┆ full_name ┆ favourite_food ┆ meal_plan ┆ age │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", + "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", + "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", + "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", + "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", + "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ 8 │\n", + "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ five │\n", + "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", + "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from skimpy import clean_columns\n", "\n", @@ -154,10 +240,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "f3c31e4a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6,)
age
i64
4
5
7
8
5
6
" + ], + "text/plain": [ + "shape: (6,)\n", + "Series: 'age' [i64]\n", + "[\n", + "\t4\n", + "\t5\n", + "\t7\n", + "\t8\n", + "\t5\n", + "\t6\n", + "]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "students = students.with_columns(pl.col(\"age\").replace(\"five\", 5).cast(pl.Int64))\n", "students[\"age\"]" @@ -181,10 +297,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "678fdd2d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6,)
meal_plan
cat
"Lunch only"
"Lunch only"
"Breakfast and lunch"
"Lunch only"
"Breakfast and lunch"
"Lunch only"
" + ], + "text/plain": [ + "shape: (6,)\n", + "Series: 'meal_plan' [cat]\n", + "[\n", + "\t\"Lunch only\"\n", + "\t\"Lunch only\"\n", + "\t\"Breakfast and lunch\"\n", + "\t\"Lunch only\"\n", + "\t\"Breakfast and lunch\"\n", + "\t\"Lunch only\"\n", + "]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "students = students.with_columns(pl.col(\"meal_plan\").cast(pl.Categorical))\n", "students[\"meal_plan\"]" @@ -202,10 +348,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "f54108d3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Schema([('student_id', Int32),\n", + " ('full_name', String),\n", + " ('favourite_food', String),\n", + " ('meal_plan', Categorical(ordering='physical')),\n", + " ('age', Int64)])" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "students = students.cast(\n", " {\"student_id\": pl.Int32, \"full_name\": pl.String, \"age\": pl.Int64}\n", @@ -243,10 +404,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "b80b958b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (19, 5)
monthyearbranditemn
stri64i64i64i64
"January"2019112343
"January"2019187219
"January"2019118222
"January"2019233331
"January"2019221569
……………
"March"2019136271
"March"2019188203
"March"2019272531
"March"2019287663
"March"2019282886
" + ], + "text/plain": [ + "shape: (19, 5)\n", + "┌─────────┬──────┬───────┬──────┬─────┐\n", + "│ month ┆ year ┆ brand ┆ item ┆ n │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n", + "╞═════════╪══════╪═══════╪══════╪═════╡\n", + "│ January ┆ 2019 ┆ 1 ┆ 1234 ┆ 3 │\n", + "│ January ┆ 2019 ┆ 1 ┆ 8721 ┆ 9 │\n", + "│ January ┆ 2019 ┆ 1 ┆ 1822 ┆ 2 │\n", + "│ January ┆ 2019 ┆ 2 ┆ 3333 ┆ 1 │\n", + "│ January ┆ 2019 ┆ 2 ┆ 2156 ┆ 9 │\n", + "│ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ March ┆ 2019 ┆ 1 ┆ 3627 ┆ 1 │\n", + "│ March ┆ 2019 ┆ 1 ┆ 8820 ┆ 3 │\n", + "│ March ┆ 2019 ┆ 2 ┆ 7253 ┆ 1 │\n", + "│ March ┆ 2019 ┆ 2 ┆ 8766 ┆ 3 │\n", + "│ March ┆ 2019 ┆ 2 ┆ 8288 ┆ 6 │\n", + "└─────────┴──────┴───────┴──────┴─────┘" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "list_of_dataframes = [\n", " pl.read_csv(x)\n", @@ -266,10 +465,57 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "4a92056c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "List of csvs is:\n", + "['data/03-sales.csv', 'data/02-sales.csv', 'data/01-sales.csv'] \n", + "\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "shape: (19, 5)
monthyearbranditemn
stri64i64i64i64
"March"2019112343
"March"2019136271
"March"2019188203
"March"2019272531
"March"2019287663
……………
"January"2019118222
"January"2019233331
"January"2019221569
"January"2019239876
"January"2019238276
" + ], + "text/plain": [ + "shape: (19, 5)\n", + "┌─────────┬──────┬───────┬──────┬─────┐\n", + "│ month ┆ year ┆ brand ┆ item ┆ n │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n", + "╞═════════╪══════╪═══════╪══════╪═════╡\n", + "│ March ┆ 2019 ┆ 1 ┆ 1234 ┆ 3 │\n", + "│ March ┆ 2019 ┆ 1 ┆ 3627 ┆ 1 │\n", + "│ March ┆ 2019 ┆ 1 ┆ 8820 ┆ 3 │\n", + "│ March ┆ 2019 ┆ 2 ┆ 7253 ┆ 1 │\n", + "│ March ┆ 2019 ┆ 2 ┆ 8766 ┆ 3 │\n", + "│ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ January ┆ 2019 ┆ 1 ┆ 1822 ┆ 2 │\n", + "│ January ┆ 2019 ┆ 2 ┆ 3333 ┆ 1 │\n", + "│ January ┆ 2019 ┆ 2 ┆ 2156 ┆ 9 │\n", + "│ January ┆ 2019 ┆ 2 ┆ 3987 ┆ 6 │\n", + "│ January ┆ 2019 ┆ 2 ┆ 3827 ┆ 6 │\n", + "└─────────┴──────┴───────┴──────┴─────┘" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import glob\n", "\n", @@ -294,7 +540,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "0bc97749", "metadata": {}, "outputs": [], @@ -312,10 +558,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "542c5223", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Schema([('student_id', Int64),\n", + " ('full_name', String),\n", + " ('favourite_food', String),\n", + " ('meal_plan', String),\n", + " ('age', Int64)])" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pl.read_csv(\"data/students-clean.csv\").schema" ] @@ -336,7 +597,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "16c6ca1b", "metadata": {}, "outputs": [], @@ -354,10 +615,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "bfd5104f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Schema([('student_id', Int32),\n", + " ('full_name', String),\n", + " ('favourite_food', String),\n", + " ('meal_plan', Categorical(ordering='physical')),\n", + " ('age', Int64)])" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pl.read_ipc(\"data/students-clean.feather\").schema" ] @@ -376,7 +652,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "227f7c50", "metadata": { "tags": [ @@ -414,7 +690,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/data-tidy.ipynb b/data-tidy.ipynb index e0ba7a7..903b0a0 100644 --- a/data-tidy.ipynb +++ b/data-tidy.ipynb @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "51a55374", "metadata": { "tags": [ @@ -111,10 +111,96 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "0f9fbf5a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " Unmelted: \n", + " first last job height weight\n", + "0 John Doe Nurse 5.5 130\n", + "1 Mary Bo Economist 6.0 150\n", + "\n", + " Melted: \n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
firstlastquantityvalue
0JohnDoeheight5.5
1MaryBoheight6.0
2JohnDoeweight130.0
3MaryBoweight150.0
\n", + "
" + ], + "text/plain": [ + " first last quantity value\n", + "0 John Doe height 5.5\n", + "1 Mary Bo height 6.0\n", + "2 John Doe weight 130.0\n", + "3 Mary Bo weight 150.0" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import pandas as pd\n", "\n", @@ -149,10 +235,71 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "bfa121cf", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
country19992000
0Afghanistan745.02666.0
1Brazil37737.080488.0
2China212258.0213766.0
\n", + "
" + ], + "text/plain": [ + " country 1999 2000\n", + "0 Afghanistan 745.0 2666.0\n", + "1 Brazil 37737.0 80488.0\n", + "2 China 212258.0 213766.0" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df_tb = pd.read_parquet(\n", " \"https://github.com/aeturrell/python4DS/raw/refs/heads/main/data/who_tb_cases.parquet\"\n", @@ -170,10 +317,92 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "dc03ccd9", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
countryyearcases
0Afghanistan1999745.0
1Brazil199937737.0
2China1999212258.0
3Afghanistan20002666.0
4Brazil200080488.0
5China2000213766.0
\n", + "
" + ], + "text/plain": [ + " country year cases\n", + "0 Afghanistan 1999 745.0\n", + "1 Brazil 1999 37737.0\n", + "2 China 1999 212258.0\n", + "3 Afghanistan 2000 2666.0\n", + "4 Brazil 2000 80488.0\n", + "5 China 2000 213766.0" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df_tb.melt(\n", " id_vars=[\"country\"],\n", @@ -203,10 +432,83 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "293768c1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
A1970A1980B1970B1980Xid
0ad2.53.22.2900010
1be1.21.3-0.5127471
2cf0.70.1-2.3662972
\n", + "
" + ], + "text/plain": [ + " A1970 A1980 B1970 B1980 X id\n", + "0 a d 2.5 3.2 2.290001 0\n", + "1 b e 1.2 1.3 -0.512747 1\n", + "2 c f 0.7 0.1 -2.366297 2" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import numpy as np\n", "\n", @@ -233,10 +535,107 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "a9ca2fa8", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
XAB
idyear
019702.290001a2.5
11970-0.512747b1.2
21970-2.366297c0.7
019802.290001d3.2
11980-0.512747e1.3
21980-2.366297f0.1
\n", + "
" + ], + "text/plain": [ + " X A B\n", + "id year \n", + "0 1970 2.290001 a 2.5\n", + "1 1970 -0.512747 b 1.2\n", + "2 1970 -2.366297 c 0.7\n", + "0 1980 2.290001 d 3.2\n", + "1 1980 -0.512747 e 1.3\n", + "2 1980 -2.366297 f 0.1" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pd.wide_to_long(df, stubnames=[\"A\", \"B\"], i=\"id\", j=\"year\")" ] @@ -261,10 +660,109 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "2b791dd1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
AB
firstsecond
barone1.3129600.427839
two-0.070985-0.738495
bazone1.042258-0.430793
two0.511735-0.782214
fooone-0.847108-1.179077
two1.0210641.015834
quxone-1.194002-0.313362
two2.226642-0.898217
\n", + "
" + ], + "text/plain": [ + " A B\n", + "first second \n", + "bar one 1.312960 0.427839\n", + " two -0.070985 -0.738495\n", + "baz one 1.042258 -0.430793\n", + " two 0.511735 -0.782214\n", + "foo one -0.847108 -1.179077\n", + " two 1.021064 1.015834\n", + "qux one -1.194002 -0.313362\n", + " two 2.226642 -0.898217" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "tuples = list(\n", " zip(\n", @@ -289,10 +787,38 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "d25eb012", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "first second \n", + "bar one A 1.312960\n", + " B 0.427839\n", + " two A -0.070985\n", + " B -0.738495\n", + "baz one A 1.042258\n", + " B -0.430793\n", + " two A 0.511735\n", + " B -0.782214\n", + "foo one A -0.847108\n", + " B -1.179077\n", + " two A 1.021064\n", + " B 1.015834\n", + "qux one A -1.194002\n", + " B -0.313362\n", + " two A 2.226642\n", + " B -0.898217\n", + "dtype: float64" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = df.stack()\n", "df" @@ -320,10 +846,95 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "b6742a54", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
firstbarbazfooqux
second
oneA1.3129601.042258-0.847108-1.194002
B0.427839-0.430793-1.179077-0.313362
twoA-0.0709850.5117351.0210642.226642
B-0.738495-0.7822141.015834-0.898217
\n", + "
" + ], + "text/plain": [ + "first bar baz foo qux\n", + "second \n", + "one A 1.312960 1.042258 -0.847108 -1.194002\n", + " B 0.427839 -0.430793 -1.179077 -0.313362\n", + "two A -0.070985 0.511735 1.021064 2.226642\n", + " B -0.738495 -0.782214 1.015834 -0.898217" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.unstack(level=0)" ] @@ -360,10 +971,91 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "fa612456", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
countryyeartypecount
0Afghanistan1999-01-01cases745
1Afghanistan1999-01-01population19987071
2Afghanistan2000-01-01cases2666
3Afghanistan2000-01-01population20595360
4Brazil1999-01-01cases37737
\n", + "
" + ], + "text/plain": [ + " country year type count\n", + "0 Afghanistan 1999-01-01 cases 745\n", + "1 Afghanistan 1999-01-01 population 19987071\n", + "2 Afghanistan 2000-01-01 cases 2666\n", + "3 Afghanistan 2000-01-01 population 20595360\n", + "4 Brazil 1999-01-01 cases 37737" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df_tb_cp = pd.read_parquet(\n", " \"https://github.com/aeturrell/python4DS/raw/refs/heads/main/data/who_tb_case_and_pop.parquet\"\n", @@ -389,10 +1081,99 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "e584cf37", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
typecountryyearcasespopulation
0Afghanistan1999-01-0174519987071
1Afghanistan2000-01-01266620595360
2Brazil1999-01-0137737172006362
3Brazil2000-01-0180488174504898
4China1999-01-012122581272915272
5China2000-01-012137661280428583
\n", + "
" + ], + "text/plain": [ + "type country year cases population\n", + "0 Afghanistan 1999-01-01 745 19987071\n", + "1 Afghanistan 2000-01-01 2666 20595360\n", + "2 Brazil 1999-01-01 37737 172006362\n", + "3 Brazil 2000-01-01 80488 174504898\n", + "4 China 1999-01-01 212258 1272915272\n", + "5 China 2000-01-01 213766 1280428583" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pivoted = df_tb_cp.pivot(\n", " index=[\"country\", \"year\"], columns=[\"type\"], values=\"count\"\n", @@ -410,10 +1191,85 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "97c6d139", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
datevariablevalue
12000-02-29A0.799993
22000-03-31A0.247382
92000-10-31A0.685868
182000-09-30B-0.911245
42000-05-31A0.874574
\n", + "
" + ], + "text/plain": [ + " date variable value\n", + "1 2000-02-29 A 0.799993\n", + "2 2000-03-31 A 0.247382\n", + "9 2000-10-31 A 0.685868\n", + "18 2000-09-30 B -0.911245\n", + "4 2000-05-31 A 0.874574" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import numpy as np\n", "\n", @@ -439,10 +1295,115 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "04f2bd28", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
variableAB
date
2000-01-31NaNNaN
2000-02-290.7116320.869024
2000-03-310.799993-0.276892
2000-04-300.247382-1.172654
2000-05-31-1.545182-1.452367
2000-06-300.874574-2.377642
2000-07-31-0.735886-2.036017
2000-08-310.0138172.211417
2000-09-30-1.401537-0.896416
2000-10-310.063176-0.911245
\n", + "
" + ], + "text/plain": [ + "variable A B\n", + "date \n", + "2000-01-31 NaN NaN\n", + "2000-02-29 0.711632 0.869024\n", + "2000-03-31 0.799993 -0.276892\n", + "2000-04-30 0.247382 -1.172654\n", + "2000-05-31 -1.545182 -1.452367\n", + "2000-06-30 0.874574 -2.377642\n", + "2000-07-31 -0.735886 -2.036017\n", + "2000-08-31 0.013817 2.211417\n", + "2000-09-30 -1.401537 -0.896416\n", + "2000-10-31 0.063176 -0.911245" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.pivot(index=\"date\", columns=\"variable\", values=\"value\").shift(1)" ] @@ -485,7 +1446,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/data-transform.ipynb b/data-transform.ipynb index 83588d0..03eed1b 100644 --- a/data-transform.ipynb +++ b/data-transform.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "04dcb195", "metadata": {}, "outputs": [], @@ -44,10 +44,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "09eb2e2e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'1.19.0'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pl.__version__" ] @@ -66,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "eff283e8", "metadata": {}, "outputs": [], @@ -97,10 +108,42 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "39f99d76", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (5, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64str
201311517515283081911"UA"1545"N14228""EWR""IAH"2271400515"2013-01-01T10:00:00Z"
201311533529485083020"UA"1714"N24211""LGA""IAH"2271416529"2013-01-01T10:00:00Z"
201311542540292385033"AA"1141"N619AA""JFK""MIA"1601089540"2013-01-01T10:00:00Z"
201311544545-110041022-18"B6"725"N804JB""JFK""BQN"1831576545"2013-01-01T10:00:00Z"
201311554600-6812837-25"DL"461"N668DN""LGA""ATL"11676260"2013-01-01T11:00:00Z"
" + ], + "text/plain": [ + "shape: (5, 19)\n", + "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬──────────────────────┐\n", + "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", + "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪══════════════════════╡\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 517 ┆ … ┆ 1400 ┆ 5 ┆ 15 ┆ 2013-01-01T10:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 533 ┆ … ┆ 1416 ┆ 5 ┆ 29 ┆ 2013-01-01T10:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 542 ┆ … ┆ 1089 ┆ 5 ┆ 40 ┆ 2013-01-01T10:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 544 ┆ … ┆ 1576 ┆ 5 ┆ 45 ┆ 2013-01-01T10:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 554 ┆ … ┆ 762 ┆ 6 ┆ 0 ┆ 2013-01-01T11:00:00Z │\n", + "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴──────────────────────┘" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights.head()" ] @@ -115,10 +158,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "95dea97b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Rows: 336776\n", + "Columns: 19\n", + "$ year 2013, 2013, 2013, 2013, 2013\n", + "$ month 1, 1, 1, 1, 1\n", + "$ day 1, 1, 1, 1, 1\n", + "$ dep_time 517, 533, 542, 544, 554\n", + "$ sched_dep_time 515, 529, 540, 545, 600\n", + "$ dep_delay 2, 4, 2, -1, -6\n", + "$ arr_time 830, 850, 923, 1004, 812\n", + "$ sched_arr_time 819, 830, 850, 1022, 837\n", + "$ arr_delay 11, 20, 33, -18, -25\n", + "$ carrier 'UA', 'UA', 'AA', 'B6', 'DL'\n", + "$ flight 1545, 1714, 1141, 725, 461\n", + "$ tailnum 'N14228', 'N24211', 'N619AA', 'N804JB', 'N668DN'\n", + "$ origin 'EWR', 'LGA', 'JFK', 'JFK', 'LGA'\n", + "$ dest 'IAH', 'IAH', 'MIA', 'BQN', 'ATL'\n", + "$ air_time 227, 227, 160, 183, 116\n", + "$ distance 1400, 1416, 1089, 1576, 762\n", + "$ hour 5, 5, 5, 5, 6\n", + "$ minute 15, 29, 40, 45, 0\n", + "$ time_hour '2013-01-01T10:00:00Z', '2013-01-01T10:00:00Z', '2013-01-01T10:00:00Z', '2013-01-01T10:00:00Z', '2013-01-01T11:00:00Z'\n", + "\n" + ] + } + ], "source": [ "flights.glimpse(max_items_per_column=5)" ] @@ -149,20 +221,93 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "ffb275b0", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (336_776,)
time_hour
str
"2013-01-01T10:00:00Z"
"2013-01-01T10:00:00Z"
"2013-01-01T10:00:00Z"
"2013-01-01T10:00:00Z"
"2013-01-01T11:00:00Z"
…
"2013-09-30T18:00:00Z"
"2013-10-01T02:00:00Z"
"2013-09-30T16:00:00Z"
"2013-09-30T15:00:00Z"
"2013-09-30T12:00:00Z"
" + ], + "text/plain": [ + "shape: (336_776,)\n", + "Series: 'time_hour' [str]\n", + "[\n", + "\t\"2013-01-01T10:00:00Z\"\n", + "\t\"2013-01-01T10:00:00Z\"\n", + "\t\"2013-01-01T10:00:00Z\"\n", + "\t\"2013-01-01T10:00:00Z\"\n", + "\t\"2013-01-01T11:00:00Z\"\n", + "\t…\n", + "\t\"2013-09-30T18:00:00Z\"\n", + "\t\"2013-10-01T02:00:00Z\"\n", + "\t\"2013-09-30T16:00:00Z\"\n", + "\t\"2013-09-30T15:00:00Z\"\n", + "\t\"2013-09-30T12:00:00Z\"\n", + "]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights.get_column(\"time_hour\")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "88a8b983", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (336_776, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64datetime[μs, UTC]
201311517515283081911"UA"1545"N14228""EWR""IAH"22714005152013-01-01 10:00:00 UTC
201311533529485083020"UA"1714"N24211""LGA""IAH"22714165292013-01-01 10:00:00 UTC
201311542540292385033"AA"1141"N619AA""JFK""MIA"16010895402013-01-01 10:00:00 UTC
201311544545-110041022-18"B6"725"N804JB""JFK""BQN"18315765452013-01-01 10:00:00 UTC
201311554600-6812837-25"DL"461"N668DN""LGA""ATL"116762602013-01-01 11:00:00 UTC
…………………………………………………
2013930null1455nullnull1634null"9E"3393null"JFK""DCA"null21314552013-09-30 18:00:00 UTC
2013930null2200nullnull2312null"9E"3525null"LGA""SYR"null1982202013-10-01 02:00:00 UTC
2013930null1210nullnull1330null"MQ"3461"N535MQ""LGA""BNA"null76412102013-09-30 16:00:00 UTC
2013930null1159nullnull1344null"MQ"3572"N511MQ""LGA""CLE"null41911592013-09-30 15:00:00 UTC
2013930null840nullnull1020null"MQ"3531"N839MQ""LGA""RDU"null4318402013-09-30 12:00:00 UTC
" + ], + "text/plain": [ + "shape: (336_776, 19)\n", + "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬─────────────────────────┐\n", + "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ datetime[μs, UTC] │\n", + "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪═════════════════════════╡\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 517 ┆ … ┆ 1400 ┆ 5 ┆ 15 ┆ 2013-01-01 10:00:00 UTC │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 533 ┆ … ┆ 1416 ┆ 5 ┆ 29 ┆ 2013-01-01 10:00:00 UTC │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 542 ┆ … ┆ 1089 ┆ 5 ┆ 40 ┆ 2013-01-01 10:00:00 UTC │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 544 ┆ … ┆ 1576 ┆ 5 ┆ 45 ┆ 2013-01-01 10:00:00 UTC │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 554 ┆ … ┆ 762 ┆ 6 ┆ 0 ┆ 2013-01-01 11:00:00 UTC │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 213 ┆ 14 ┆ 55 ┆ 2013-09-30 18:00:00 UTC │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 198 ┆ 22 ┆ 0 ┆ 2013-10-01 02:00:00 UTC │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 764 ┆ 12 ┆ 10 ┆ 2013-09-30 16:00:00 UTC │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 419 ┆ 11 ┆ 59 ┆ 2013-09-30 15:00:00 UTC │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 431 ┆ 8 ┆ 40 ┆ 2013-09-30 12:00:00 UTC │\n", + "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴─────────────────────────┘" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights.with_columns(pl.col(\"time_hour\").str.to_datetime())" ] @@ -187,10 +332,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "1b6bd8b1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (4, 5)
col0col1col2col3col4
i64i64i64strstr
000"a""alpha"
000"b""gamma"
000"b""gamma"
000"a""gamma"
" + ], + "text/plain": [ + "shape: (4, 5)\n", + "┌──────┬──────┬──────┬──────┬───────┐\n", + "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ str ┆ str │\n", + "╞══════╪══════╪══════╪══════╪═══════╡\n", + "│ 0 ┆ 0 ┆ 0 ┆ a ┆ alpha │\n", + "│ 0 ┆ 0 ┆ 0 ┆ b ┆ gamma │\n", + "│ 0 ┆ 0 ┆ 0 ┆ b ┆ gamma │\n", + "│ 0 ┆ 0 ┆ 0 ┆ a ┆ gamma │\n", + "└──────┴──────┴──────┴──────┴───────┘" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = pl.DataFrame(\n", " data={\n", @@ -229,10 +405,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "cb114649", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (365, 4)
yearmonthdayarr_delay
i64i64i64f64
2013102-19.47619
201342630.380952
20131129-25.384615
2013430-9.52381
201381614.05
…………
20135151.095238
20137912.714286
2013531-6.52381
201311235.375
2013416-1.736842
" + ], + "text/plain": [ + "shape: (365, 4)\n", + "┌──────┬───────┬─────┬────────────┐\n", + "│ year ┆ month ┆ day ┆ arr_delay │\n", + "│ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ f64 │\n", + "╞══════╪═══════╪═════╪════════════╡\n", + "│ 2013 ┆ 10 ┆ 2 ┆ -19.47619 │\n", + "│ 2013 ┆ 4 ┆ 26 ┆ 30.380952 │\n", + "│ 2013 ┆ 11 ┆ 29 ┆ -25.384615 │\n", + "│ 2013 ┆ 4 ┆ 30 ┆ -9.52381 │\n", + "│ 2013 ┆ 8 ┆ 16 ┆ 14.05 │\n", + "│ … ┆ … ┆ … ┆ … │\n", + "│ 2013 ┆ 5 ┆ 15 ┆ 1.095238 │\n", + "│ 2013 ┆ 7 ┆ 9 ┆ 12.714286 │\n", + "│ 2013 ┆ 5 ┆ 31 ┆ -6.52381 │\n", + "│ 2013 ┆ 11 ┆ 23 ┆ 5.375 │\n", + "│ 2013 ┆ 4 ┆ 16 ┆ -1.736842 │\n", + "└──────┴───────┴─────┴────────────┘" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights.filter(pl.col(\"dest\") == \"IAH\").group_by([\"year\", \"month\", \"day\"]).agg(\n", " pl.col(\"arr_delay\").mean()\n", @@ -267,10 +481,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "3958ddb5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 7)
col0col1col2col3col4col5col6
i64i64i64i64i64i64str
012345"apple"
67891011"orange"
121314151617"pineapple"
181920212223"mango"
242526272829"kiwi"
303132333435"lemon"
" + ], + "text/plain": [ + "shape: (6, 7)\n", + "┌──────┬──────┬──────┬──────┬──────┬──────┬───────────┐\n", + "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 ┆ col6 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", + "╞══════╪══════╪══════╪══════╪══════╪══════╪═══════════╡\n", + "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ 4 ┆ 5 ┆ apple │\n", + "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ 10 ┆ 11 ┆ orange │\n", + "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ 16 ┆ 17 ┆ pineapple │\n", + "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ 22 ┆ 23 ┆ mango │\n", + "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 ┆ kiwi │\n", + "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ 34 ┆ 35 ┆ lemon │\n", + "└──────┴──────┴──────┴──────┴──────┴──────┴───────────┘" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import numpy as np\n", "\n", @@ -299,10 +546,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "0c2faf83", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(24, 25, 26, 27, 28, 29, 'kiwi')" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Gets the first row of the DataFrame\n", "df.row(0)\n", @@ -321,10 +579,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "9d34599b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(18, 19, 20, 21, 22, 23, 'mango')" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.row(by_predicate=pl.col(\"col6\") == \"mango\")" ] @@ -339,10 +608,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "9bf6313b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'col0': 18,\n", + " 'col1': 19,\n", + " 'col2': 20,\n", + " 'col3': 21,\n", + " 'col4': 22,\n", + " 'col5': 23,\n", + " 'col6': 'mango'}" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Get the first row of the DataFrame as a dictionary\n", "df.row(0, named=True)\n", @@ -361,10 +647,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "f7e8e892", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (2, 7)
col0col1col2col3col4col5col6
i64i64i64i64i64i64str
242526272829"kiwi"
303132333435"lemon"
" + ], + "text/plain": [ + "shape: (2, 7)\n", + "┌──────┬──────┬──────┬──────┬──────┬──────┬───────┐\n", + "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 ┆ col6 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", + "╞══════╪══════╪══════╪══════╪══════╪══════╪═══════╡\n", + "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 ┆ kiwi │\n", + "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ 34 ┆ 35 ┆ lemon │\n", + "└──────┴──────┴──────┴──────┴──────┴──────┴───────┘" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.slice(-2, 2)" ] @@ -381,10 +696,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "54a9d2b1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (2, 7)
col0col1col2col3col4col5col6
i64i64i64i64i64i64str
121314151617"pineapple"
242526272829"kiwi"
" + ], + "text/plain": [ + "shape: (2, 7)\n", + "┌──────┬──────┬──────┬──────┬──────┬──────┬───────────┐\n", + "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 ┆ col6 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", + "╞══════╪══════╪══════╪══════╪══════╪══════╪═══════════╡\n", + "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ 16 ┆ 17 ┆ pineapple │\n", + "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 ┆ kiwi │\n", + "└──────┴──────┴──────┴──────┴──────┴──────┴───────────┘" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.filter((pl.col(\"col6\") == \"kiwi\") | (pl.col(\"col6\") == \"pineapple\"))" ] @@ -399,10 +743,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "7849a962", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (4, 7)
col0col1col2col3col4col5col6
i64i64i64i64i64i64str
121314151617"pineapple"
181920212223"mango"
242526272829"kiwi"
303132333435"lemon"
" + ], + "text/plain": [ + "shape: (4, 7)\n", + "┌──────┬──────┬──────┬──────┬──────┬──────┬───────────┐\n", + "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 ┆ col6 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", + "╞══════╪══════╪══════╪══════╪══════╪══════╪═══════════╡\n", + "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ 16 ┆ 17 ┆ pineapple │\n", + "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ 22 ┆ 23 ┆ mango │\n", + "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 ┆ kiwi │\n", + "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ 34 ┆ 35 ┆ lemon │\n", + "└──────┴──────┴──────┴──────┴──────┴──────┴───────────┘" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.filter(pl.col(\"col0\") > 6)" ] @@ -417,10 +792,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "c6dd919f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (4_334, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64str
201311517515283081911"UA"1545"N14228""EWR""IAH"2271400515"2013-01-01T10:00:00Z"
201311533529485083020"UA"1714"N24211""LGA""IAH"2271416529"2013-01-01T10:00:00Z"
201311542540292385033"AA"1141"N619AA""JFK""MIA"1601089540"2013-01-01T10:00:00Z"
201311544545-110041022-18"B6"725"N804JB""JFK""BQN"1831576545"2013-01-01T10:00:00Z"
201311554600-6812837-25"DL"461"N668DN""LGA""ATL"11676260"2013-01-01T11:00:00Z"
…………………………………………………
20131523552359-4425442-17"B6"707"N583JB""JFK""SJU"19315982359"2013-01-06T04:00:00Z"
20131523572359-2432437-5"B6"727"N649JB""JFK""BQN"19515762359"2013-01-06T04:00:00Z"
201315null1400nullnull1518null"EV"5712"N827AS""JFK""IAD"null228140"2013-01-05T19:00:00Z"
201315null840nullnull1001null"9E"3422null"JFK""BOS"null187840"2013-01-05T13:00:00Z"
201315null1430nullnull1735null"AA"883"N544AA""EWR""DFW"null13721430"2013-01-05T19:00:00Z"
" + ], + "text/plain": [ + "shape: (4_334, 19)\n", + "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬──────────────────────┐\n", + "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", + "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪══════════════════════╡\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 517 ┆ … ┆ 1400 ┆ 5 ┆ 15 ┆ 2013-01-01T10:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 533 ┆ … ┆ 1416 ┆ 5 ┆ 29 ┆ 2013-01-01T10:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 542 ┆ … ┆ 1089 ┆ 5 ┆ 40 ┆ 2013-01-01T10:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 544 ┆ … ┆ 1576 ┆ 5 ┆ 45 ┆ 2013-01-01T10:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 554 ┆ … ┆ 762 ┆ 6 ┆ 0 ┆ 2013-01-01T11:00:00Z │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ 2013 ┆ 1 ┆ 5 ┆ 2355 ┆ … ┆ 1598 ┆ 23 ┆ 59 ┆ 2013-01-06T04:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 5 ┆ 2357 ┆ … ┆ 1576 ┆ 23 ┆ 59 ┆ 2013-01-06T04:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 5 ┆ null ┆ … ┆ 228 ┆ 14 ┆ 0 ┆ 2013-01-05T19:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 5 ┆ null ┆ … ┆ 187 ┆ 8 ┆ 40 ┆ 2013-01-05T13:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 5 ┆ null ┆ … ┆ 1372 ┆ 14 ┆ 30 ┆ 2013-01-05T19:00:00Z │\n", + "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴──────────────────────┘" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Flights that departed on January 1\n", "flights.filter((pl.col(\"month\") == 1) & (pl.col(\"day\") <= 5))" @@ -446,20 +859,96 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "395c9c62", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (336_776, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64str
201311null1630nullnull1815null"EV"4308"N18120""EWR""RDU"null4161630"2013-01-01T21:00:00Z"
201311null1935nullnull2240null"AA"791"N3EHAA""LGA""DFW"null13891935"2013-01-02T00:00:00Z"
201311null1500nullnull1825null"AA"1925"N3EVAA""LGA""MIA"null1096150"2013-01-01T20:00:00Z"
201311null600nullnull901null"B6"125"N618JB""JFK""FLL"null106960"2013-01-01T11:00:00Z"
201312null1540nullnull1747null"EV"4352"N10575""EWR""CVG"null5691540"2013-01-02T20:00:00Z"
…………………………………………………
201372824002359141134427"B6"1503"N503JB""JFK""SJU"20415982359"2013-07-29T03:00:00Z"
20138102400224575110169"B6"234"N328JB""JFK""BTV"532662245"2013-08-11T02:00:00Z"
20138202400235913543504"B6"745"N708JB""JFK""PSE"20116172359"2013-08-21T03:00:00Z"
20139224002359141134031"B6"839"N828JB""JFK""BQN"21715762359"2013-09-03T03:00:00Z"
2013912240020002402032230213"DL"1147"N910DE""LGA""ATL"101762200"2013-09-13T00:00:00Z"
" + ], + "text/plain": [ + "shape: (336_776, 19)\n", + "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬──────────────────────┐\n", + "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", + "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪══════════════════════╡\n", + "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 416 ┆ 16 ┆ 30 ┆ 2013-01-01T21:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1389 ┆ 19 ┆ 35 ┆ 2013-01-02T00:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1096 ┆ 15 ┆ 0 ┆ 2013-01-01T20:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1069 ┆ 6 ┆ 0 ┆ 2013-01-01T11:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 2 ┆ null ┆ … ┆ 569 ┆ 15 ┆ 40 ┆ 2013-01-02T20:00:00Z │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ 2013 ┆ 7 ┆ 28 ┆ 2400 ┆ … ┆ 1598 ┆ 23 ┆ 59 ┆ 2013-07-29T03:00:00Z │\n", + "│ 2013 ┆ 8 ┆ 10 ┆ 2400 ┆ … ┆ 266 ┆ 22 ┆ 45 ┆ 2013-08-11T02:00:00Z │\n", + "│ 2013 ┆ 8 ┆ 20 ┆ 2400 ┆ … ┆ 1617 ┆ 23 ┆ 59 ┆ 2013-08-21T03:00:00Z │\n", + "│ 2013 ┆ 9 ┆ 2 ┆ 2400 ┆ … ┆ 1576 ┆ 23 ┆ 59 ┆ 2013-09-03T03:00:00Z │\n", + "│ 2013 ┆ 9 ┆ 12 ┆ 2400 ┆ … ┆ 762 ┆ 20 ┆ 0 ┆ 2013-09-13T00:00:00Z │\n", + "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴──────────────────────┘" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights.sort(\"dep_time\")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "9d5f4270", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (336_776, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64str
201311null1935nullnull2240null"AA"791"N3EHAA""LGA""DFW"null13891935"2013-01-02T00:00:00Z"
201311null1500nullnull1825null"AA"1925"N3EVAA""LGA""MIA"null1096150"2013-01-01T20:00:00Z"
201311null1630nullnull1815null"EV"4308"N18120""EWR""RDU"null4161630"2013-01-01T21:00:00Z"
201311null600nullnull901null"B6"125"N618JB""JFK""FLL"null106960"2013-01-01T11:00:00Z"
201311517515283081911"UA"1545"N14228""EWR""IAH"2271400515"2013-01-01T10:00:00Z"
…………………………………………………
20131231232122503146838"B6"2002"N179JB""JFK""BUF"663012250"2014-01-01T03:00:00Z"
2013123123282330-24124093"B6"1389"N651JB""EWR""SJU"19816082330"2014-01-01T04:00:00Z"
20131231233222454758355"B6"486"N334JB""JFK""ROC"602642245"2014-01-01T03:00:00Z"
2013123123552359-4430440-10"B6"1503"N509JB""JFK""SJU"19515982359"2014-01-01T04:00:00Z"
2013123123562359-3436445-9"B6"745"N665JB""JFK""PSE"20016172359"2014-01-01T04:00:00Z"
" + ], + "text/plain": [ + "shape: (336_776, 19)\n", + "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬──────────────────────┐\n", + "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", + "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪══════════════════════╡\n", + "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1389 ┆ 19 ┆ 35 ┆ 2013-01-02T00:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1096 ┆ 15 ┆ 0 ┆ 2013-01-01T20:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 416 ┆ 16 ┆ 30 ┆ 2013-01-01T21:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1069 ┆ 6 ┆ 0 ┆ 2013-01-01T11:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 517 ┆ … ┆ 1400 ┆ 5 ┆ 15 ┆ 2013-01-01T10:00:00Z │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ 2013 ┆ 12 ┆ 31 ┆ 2321 ┆ … ┆ 301 ┆ 22 ┆ 50 ┆ 2014-01-01T03:00:00Z │\n", + "│ 2013 ┆ 12 ┆ 31 ┆ 2328 ┆ … ┆ 1608 ┆ 23 ┆ 30 ┆ 2014-01-01T04:00:00Z │\n", + "│ 2013 ┆ 12 ┆ 31 ┆ 2332 ┆ … ┆ 264 ┆ 22 ┆ 45 ┆ 2014-01-01T03:00:00Z │\n", + "│ 2013 ┆ 12 ┆ 31 ┆ 2355 ┆ … ┆ 1598 ┆ 23 ┆ 59 ┆ 2014-01-01T04:00:00Z │\n", + "│ 2013 ┆ 12 ┆ 31 ┆ 2356 ┆ … ┆ 1617 ┆ 23 ┆ 59 ┆ 2014-01-01T04:00:00Z │\n", + "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴──────────────────────┘" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Sort by multiple columns by passing a list of columns.\n", "flights.sort([\"year\", \"month\", \"day\", \"dep_time\"])\n", @@ -479,20 +968,96 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "483acdc1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (336_776, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64str
201311null1630nullnull1815null"EV"4308"N18120""EWR""RDU"null4161630"2013-01-01T21:00:00Z"
201311null1935nullnull2240null"AA"791"N3EHAA""LGA""DFW"null13891935"2013-01-02T00:00:00Z"
201311null1500nullnull1825null"AA"1925"N3EVAA""LGA""MIA"null1096150"2013-01-01T20:00:00Z"
201311null600nullnull901null"B6"125"N618JB""JFK""FLL"null106960"2013-01-01T11:00:00Z"
201312null1540nullnull1747null"EV"4352"N10575""EWR""CVG"null5691540"2013-01-02T20:00:00Z"
…………………………………………………
201312917031730-2719471957-10"F9"837"N208FR""LGA""DEN"25016201730"2013-01-29T22:00:00Z"
201311119001930-3022332243-10"DL"1435"N934DL""LGA""TPA"13910101930"2013-01-12T00:00:00Z"
2013111014081440-3215491559-10"EV"5713"N825AS""LGA""IAD"522291440"2013-11-10T19:00:00Z"
20132320222055-3322402338-58"DL"1715"N612DL""LGA""MSY"16211832055"2013-02-04T01:00:00Z"
201312720402123-4340235248"B6"97"N592JB""JFK""DEN"26516262123"2013-12-08T02:00:00Z"
" + ], + "text/plain": [ + "shape: (336_776, 19)\n", + "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬──────────────────────┐\n", + "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", + "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪══════════════════════╡\n", + "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 416 ┆ 16 ┆ 30 ┆ 2013-01-01T21:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1389 ┆ 19 ┆ 35 ┆ 2013-01-02T00:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1096 ┆ 15 ┆ 0 ┆ 2013-01-01T20:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1069 ┆ 6 ┆ 0 ┆ 2013-01-01T11:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 2 ┆ null ┆ … ┆ 569 ┆ 15 ┆ 40 ┆ 2013-01-02T20:00:00Z │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ 2013 ┆ 1 ┆ 29 ┆ 1703 ┆ … ┆ 1620 ┆ 17 ┆ 30 ┆ 2013-01-29T22:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 11 ┆ 1900 ┆ … ┆ 1010 ┆ 19 ┆ 30 ┆ 2013-01-12T00:00:00Z │\n", + "│ 2013 ┆ 11 ┆ 10 ┆ 1408 ┆ … ┆ 229 ┆ 14 ┆ 40 ┆ 2013-11-10T19:00:00Z │\n", + "│ 2013 ┆ 2 ┆ 3 ┆ 2022 ┆ … ┆ 1183 ┆ 20 ┆ 55 ┆ 2013-02-04T01:00:00Z │\n", + "│ 2013 ┆ 12 ┆ 7 ┆ 2040 ┆ … ┆ 1626 ┆ 21 ┆ 23 ┆ 2013-12-08T02:00:00Z │\n", + "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴──────────────────────┘" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights.sort(\"dep_delay\", descending=True)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "80bf3df7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (336_776, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64str
201338null1800nullnull1937null"UA"1177null"LGA""ORD"null733180"2013-03-08T23:00:00Z"
201328null1659nullnull1822null"UA"531null"EWR""BOS"null2001659"2013-02-08T21:00:00Z"
2013813null1727nullnull1941null"EV"5892"N16561""EWR""CVG"null5691727"2013-08-13T21:00:00Z"
2013813null1225nullnull1338null"EV"5897"N15973""EWR""ORF"null2841225"2013-08-13T16:00:00Z"
2013930null840nullnull1020null"MQ"3531"N839MQ""LGA""RDU"null431840"2013-09-30T12:00:00Z"
…………………………………………………
201312917031730-2719471957-10"F9"837"N208FR""LGA""DEN"25016201730"2013-01-29T22:00:00Z"
201311119001930-3022332243-10"DL"1435"N934DL""LGA""TPA"13910101930"2013-01-12T00:00:00Z"
2013111014081440-3215491559-10"EV"5713"N825AS""LGA""IAD"522291440"2013-11-10T19:00:00Z"
20132320222055-3322402338-58"DL"1715"N612DL""LGA""MSY"16211832055"2013-02-04T01:00:00Z"
201312720402123-4340235248"B6"97"N592JB""JFK""DEN"26516262123"2013-12-08T02:00:00Z"
" + ], + "text/plain": [ + "shape: (336_776, 19)\n", + "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬──────────────────────┐\n", + "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", + "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪══════════════════════╡\n", + "│ 2013 ┆ 3 ┆ 8 ┆ null ┆ … ┆ 733 ┆ 18 ┆ 0 ┆ 2013-03-08T23:00:00Z │\n", + "│ 2013 ┆ 2 ┆ 8 ┆ null ┆ … ┆ 200 ┆ 16 ┆ 59 ┆ 2013-02-08T21:00:00Z │\n", + "│ 2013 ┆ 8 ┆ 13 ┆ null ┆ … ┆ 569 ┆ 17 ┆ 27 ┆ 2013-08-13T21:00:00Z │\n", + "│ 2013 ┆ 8 ┆ 13 ┆ null ┆ … ┆ 284 ┆ 12 ┆ 25 ┆ 2013-08-13T16:00:00Z │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 431 ┆ 8 ┆ 40 ┆ 2013-09-30T12:00:00Z │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ 2013 ┆ 1 ┆ 29 ┆ 1703 ┆ … ┆ 1620 ┆ 17 ┆ 30 ┆ 2013-01-29T22:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 11 ┆ 1900 ┆ … ┆ 1010 ┆ 19 ┆ 30 ┆ 2013-01-12T00:00:00Z │\n", + "│ 2013 ┆ 11 ┆ 10 ┆ 1408 ┆ … ┆ 229 ┆ 14 ┆ 40 ┆ 2013-11-10T19:00:00Z │\n", + "│ 2013 ┆ 2 ┆ 3 ┆ 2022 ┆ … ┆ 1183 ┆ 20 ┆ 55 ┆ 2013-02-04T01:00:00Z │\n", + "│ 2013 ┆ 12 ┆ 7 ┆ 2040 ┆ … ┆ 1626 ┆ 21 ┆ 23 ┆ 2013-12-08T02:00:00Z │\n", + "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴──────────────────────┘" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights.sort([\"dep_delay\", \"arr_delay\"], descending=[True, False])" ] @@ -508,10 +1073,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "a939f3c2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (3, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64str
20131115251530-519341805null"MQ"4525"N719MQ""LGA""XNA"null11471530"2013-01-01T20:00:00Z"
20131117401745-521582020null"MQ"4413"N739MQ""LGA""XNA"null11471745"2013-01-01T22:00:00Z"
20131218481840823332151null"9E"3325"N920XJ""JFK""DFW"null13911840"2013-01-02T23:00:00Z"
" + ], + "text/plain": [ + "shape: (3, 19)\n", + "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬──────────────────────┐\n", + "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", + "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪══════════════════════╡\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 1525 ┆ … ┆ 1147 ┆ 15 ┆ 30 ┆ 2013-01-01T20:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 1740 ┆ … ┆ 1147 ┆ 17 ┆ 45 ┆ 2013-01-01T22:00:00Z │\n", + "│ 2013 ┆ 1 ┆ 2 ┆ 1848 ┆ … ┆ 1391 ┆ 18 ┆ 40 ┆ 2013-01-02T23:00:00Z │\n", + "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴──────────────────────┘" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " flights.filter((pl.col(\"dep_delay\") <= 10) & (pl.col(\"dep_delay\") >= -10))\n", @@ -572,10 +1167,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "86827cf9", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 8)
col0col1col2col3col4col5col6new_column0
i64i64i64i64i64i64stri32
012345"apple"5
67891011"orange"5
121314151617"pineapple"5
181920212223"mango"5
242526272829"kiwi"5
303132333435"lemon"5
" + ], + "text/plain": [ + "shape: (6, 8)\n", + "┌──────┬──────┬──────┬──────┬──────┬──────┬───────────┬─────────────┐\n", + "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 ┆ col6 ┆ new_column0 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ str ┆ i32 │\n", + "╞══════╪══════╪══════╪══════╪══════╪══════╪═══════════╪═════════════╡\n", + "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ 4 ┆ 5 ┆ apple ┆ 5 │\n", + "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ 10 ┆ 11 ┆ orange ┆ 5 │\n", + "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ 16 ┆ 17 ┆ pineapple ┆ 5 │\n", + "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ 22 ┆ 23 ┆ mango ┆ 5 │\n", + "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 ┆ kiwi ┆ 5 │\n", + "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ 34 ┆ 35 ┆ lemon ┆ 5 │\n", + "└──────┴──────┴──────┴──────┴──────┴──────┴───────────┴─────────────┘" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = df.with_columns(new_column0=pl.lit(5))\n", "df" @@ -591,10 +1219,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "0ab01f9d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 8)
col0col1col2col3col4col5col6new_column0
i64i64i64i64i64i64stri64
012345"apple"0
67891011"orange"1
121314151617"pineapple"2
181920212223"mango"3
242526272829"kiwi"4
303132333435"lemon"5
" + ], + "text/plain": [ + "shape: (6, 8)\n", + "┌──────┬──────┬──────┬──────┬──────┬──────┬───────────┬─────────────┐\n", + "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 ┆ col6 ┆ new_column0 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ str ┆ i64 │\n", + "╞══════╪══════╪══════╪══════╪══════╪══════╪═══════════╪═════════════╡\n", + "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ 4 ┆ 5 ┆ apple ┆ 0 │\n", + "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ 10 ┆ 11 ┆ orange ┆ 1 │\n", + "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ 16 ┆ 17 ┆ pineapple ┆ 2 │\n", + "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ 22 ┆ 23 ┆ mango ┆ 3 │\n", + "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 ┆ kiwi ┆ 4 │\n", + "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ 34 ┆ 35 ┆ lemon ┆ 5 │\n", + "└──────┴──────┴──────┴──────┴──────┴──────┴───────────┴─────────────┘" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = df.with_columns(new_column0=pl.Series([0, 1, 2, 3, 4, 5]))\n", "df" @@ -614,10 +1275,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "id": "fff10e83", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 10)
col0col1col2col3col4col5col6new_column0new_column1new_column2
i64i64i64i64i64i64stri64i32i32
012345"apple"056
67891011"orange"156
121314151617"pineapple"256
181920212223"mango"356
242526272829"kiwi"456
303132333435"lemon"556
" + ], + "text/plain": [ + "shape: (6, 10)\n", + "┌──────┬──────┬──────┬──────┬───┬───────────┬─────────────┬─────────────┬─────────────┐\n", + "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ … ┆ col6 ┆ new_column0 ┆ new_column1 ┆ new_column2 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ str ┆ i64 ┆ i32 ┆ i32 │\n", + "╞══════╪══════╪══════╪══════╪═══╪═══════════╪═════════════╪═════════════╪═════════════╡\n", + "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ … ┆ apple ┆ 0 ┆ 5 ┆ 6 │\n", + "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ … ┆ orange ┆ 1 ┆ 5 ┆ 6 │\n", + "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ … ┆ pineapple ┆ 2 ┆ 5 ┆ 6 │\n", + "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ … ┆ mango ┆ 3 ┆ 5 ┆ 6 │\n", + "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ … ┆ kiwi ┆ 4 ┆ 5 ┆ 6 │\n", + "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ … ┆ lemon ┆ 5 ┆ 5 ┆ 6 │\n", + "└──────┴──────┴──────┴──────┴───┴───────────┴─────────────┴─────────────┴─────────────┘" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = df.with_columns(new_column1=pl.lit(5), new_column2=pl.lit(6))\n", "df" @@ -633,10 +1327,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "id": "82477100", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 10)
col0col1col2col3col4col5col6new_column0new_column1new_column2
i64i64i64i64i64i64stri64i32i64
012345"apple"050
67891011"orange"155
121314151617"pineapple"2510
181920212223"mango"3515
242526272829"kiwi"4520
303132333435"lemon"5525
" + ], + "text/plain": [ + "shape: (6, 10)\n", + "┌──────┬──────┬──────┬──────┬───┬───────────┬─────────────┬─────────────┬─────────────┐\n", + "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ … ┆ col6 ┆ new_column0 ┆ new_column1 ┆ new_column2 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ str ┆ i64 ┆ i32 ┆ i64 │\n", + "╞══════╪══════╪══════╪══════╪═══╪═══════════╪═════════════╪═════════════╪═════════════╡\n", + "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ … ┆ apple ┆ 0 ┆ 5 ┆ 0 │\n", + "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ … ┆ orange ┆ 1 ┆ 5 ┆ 5 │\n", + "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ … ┆ pineapple ┆ 2 ┆ 5 ┆ 10 │\n", + "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ … ┆ mango ┆ 3 ┆ 5 ┆ 15 │\n", + "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ … ┆ kiwi ┆ 4 ┆ 5 ┆ 20 │\n", + "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ … ┆ lemon ┆ 5 ┆ 5 ┆ 25 │\n", + "└──────┴──────┴──────┴──────┴───┴───────────┴─────────────┴─────────────┴─────────────┘" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.with_columns(new_column2=pl.col(\"col0\") - pl.col(\"new_column0\"))" ] @@ -652,10 +1379,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "55645bdd", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (336_776, 21)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hourgainspeed
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64stri64f64
201311517515283081911"UA"1545"N14228""EWR""IAH"2271400515"2013-01-01T10:00:00Z"-9370.044053
201311533529485083020"UA"1714"N24211""LGA""IAH"2271416529"2013-01-01T10:00:00Z"-16374.273128
201311542540292385033"AA"1141"N619AA""JFK""MIA"1601089540"2013-01-01T10:00:00Z"-31408.375
201311544545-110041022-18"B6"725"N804JB""JFK""BQN"1831576545"2013-01-01T10:00:00Z"17516.721311
201311554600-6812837-25"DL"461"N668DN""LGA""ATL"11676260"2013-01-01T11:00:00Z"19394.137931
………………………………………………………
2013930null1455nullnull1634null"9E"3393null"JFK""DCA"null2131455"2013-09-30T18:00:00Z"nullnull
2013930null2200nullnull2312null"9E"3525null"LGA""SYR"null198220"2013-10-01T02:00:00Z"nullnull
2013930null1210nullnull1330null"MQ"3461"N535MQ""LGA""BNA"null7641210"2013-09-30T16:00:00Z"nullnull
2013930null1159nullnull1344null"MQ"3572"N511MQ""LGA""CLE"null4191159"2013-09-30T15:00:00Z"nullnull
2013930null840nullnull1020null"MQ"3531"N839MQ""LGA""RDU"null431840"2013-09-30T12:00:00Z"nullnull
" + ], + "text/plain": [ + "shape: (336_776, 21)\n", + "┌──────┬───────┬─────┬──────────┬───┬────────┬──────────────────────┬──────┬────────────┐\n", + "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ minute ┆ time_hour ┆ gain ┆ speed │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ str ┆ i64 ┆ f64 │\n", + "╞══════╪═══════╪═════╪══════════╪═══╪════════╪══════════════════════╪══════╪════════════╡\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 517 ┆ … ┆ 15 ┆ 2013-01-01T10:00:00Z ┆ -9 ┆ 370.044053 │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 533 ┆ … ┆ 29 ┆ 2013-01-01T10:00:00Z ┆ -16 ┆ 374.273128 │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 542 ┆ … ┆ 40 ┆ 2013-01-01T10:00:00Z ┆ -31 ┆ 408.375 │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 544 ┆ … ┆ 45 ┆ 2013-01-01T10:00:00Z ┆ 17 ┆ 516.721311 │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 554 ┆ … ┆ 0 ┆ 2013-01-01T11:00:00Z ┆ 19 ┆ 394.137931 │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 55 ┆ 2013-09-30T18:00:00Z ┆ null ┆ null │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 0 ┆ 2013-10-01T02:00:00Z ┆ null ┆ null │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 10 ┆ 2013-09-30T16:00:00Z ┆ null ┆ null │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 59 ┆ 2013-09-30T15:00:00Z ┆ null ┆ null │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 40 ┆ 2013-09-30T12:00:00Z ┆ null ┆ null │\n", + "└──────┴───────┴─────┴──────────┴───┴────────┴──────────────────────┴──────┴────────────┘" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights.with_columns(\n", " (pl.col(\"dep_delay\") - pl.col(\"arr_delay\")).alias(\"gain\"),\n", @@ -681,10 +1446,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "4643b978", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6,)
col0
i64
0
6
12
18
24
30
" + ], + "text/plain": [ + "shape: (6,)\n", + "Series: 'col0' [i64]\n", + "[\n", + "\t0\n", + "\t6\n", + "\t12\n", + "\t18\n", + "\t24\n", + "\t30\n", + "]" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[\"col0\"]" ] @@ -699,10 +1494,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "id": "219852d8", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 3)
col0new_column0col2
i64i64i64
002
618
12214
18320
24426
30532
" + ], + "text/plain": [ + "shape: (6, 3)\n", + "┌──────┬─────────────┬──────┐\n", + "│ col0 ┆ new_column0 ┆ col2 │\n", + "│ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 │\n", + "╞══════╪═════════════╪══════╡\n", + "│ 0 ┆ 0 ┆ 2 │\n", + "│ 6 ┆ 1 ┆ 8 │\n", + "│ 12 ┆ 2 ┆ 14 │\n", + "│ 18 ┆ 3 ┆ 20 │\n", + "│ 24 ┆ 4 ┆ 26 │\n", + "│ 30 ┆ 5 ┆ 32 │\n", + "└──────┴─────────────┴──────┘" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[[\"col0\", \"new_column0\", \"col2\"]]" ] @@ -717,10 +1545,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "id": "1bc0cd22", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 3)
col1col2col3
i64i64i64
002
618
12214
18320
24426
30532
" + ], + "text/plain": [ + "shape: (6, 3)\n", + "┌──────┬──────┬──────┐\n", + "│ col1 ┆ col2 ┆ col3 │\n", + "│ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 │\n", + "╞══════╪══════╪══════╡\n", + "│ 0 ┆ 0 ┆ 2 │\n", + "│ 6 ┆ 1 ┆ 8 │\n", + "│ 12 ┆ 2 ┆ 14 │\n", + "│ 18 ┆ 3 ┆ 20 │\n", + "│ 24 ┆ 4 ┆ 26 │\n", + "│ 30 ┆ 5 ┆ 32 │\n", + "└──────┴──────┴──────┘" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# selecting a single column\n", "df.select(\"col0\")\n", @@ -742,10 +1603,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "id": "ed447fb7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 3)
col0new_column0col2
i64i64i64
024
6316
12428
18540
24652
30764
" + ], + "text/plain": [ + "shape: (6, 3)\n", + "┌──────┬─────────────┬──────┐\n", + "│ col0 ┆ new_column0 ┆ col2 │\n", + "│ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 │\n", + "╞══════╪═════════════╪══════╡\n", + "│ 0 ┆ 2 ┆ 4 │\n", + "│ 6 ┆ 3 ┆ 16 │\n", + "│ 12 ┆ 4 ┆ 28 │\n", + "│ 18 ┆ 5 ┆ 40 │\n", + "│ 24 ┆ 6 ┆ 52 │\n", + "│ 30 ┆ 7 ┆ 64 │\n", + "└──────┴─────────────┴──────┘" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.select(pl.col(\"col0\"), pl.col(\"new_column0\") + 2, pl.col(\"col2\") * 2)" ] @@ -760,10 +1654,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "id": "eabfd313", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (2, 3)
col0new_column0col2
i64i64i64
002
618
" + ], + "text/plain": [ + "shape: (2, 3)\n", + "┌──────┬─────────────┬──────┐\n", + "│ col0 ┆ new_column0 ┆ col2 │\n", + "│ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 │\n", + "╞══════╪═════════════╪══════╡\n", + "│ 0 ┆ 0 ┆ 2 │\n", + "│ 6 ┆ 1 ┆ 8 │\n", + "└──────┴─────────────┴──────┘" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.select(\"col0\", \"new_column0\", \"col2\").slice(0, 2)" ] @@ -778,10 +1701,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "id": "aed67406", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (336_776, 14)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delayflightair_timedistancehourminute
i64i64i64i64i64i64i64i64i64i64i64i64i64i64
20131151751528308191115452271400515
20131153352948508302017142271416529
20131154254029238503311411601089540
201311544545-110041022-187251831576545
201311554600-6812837-2546111676260
……………………………………
2013930null1455nullnull1634null3393null2131455
2013930null2200nullnull2312null3525null198220
2013930null1210nullnull1330null3461null7641210
2013930null1159nullnull1344null3572null4191159
2013930null840nullnull1020null3531null431840
" + ], + "text/plain": [ + "shape: (336_776, 14)\n", + "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────────┬──────┬────────┐\n", + "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ air_time ┆ distance ┆ hour ┆ minute │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n", + "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════════╪══════╪════════╡\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 517 ┆ … ┆ 227 ┆ 1400 ┆ 5 ┆ 15 │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 533 ┆ … ┆ 227 ┆ 1416 ┆ 5 ┆ 29 │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 542 ┆ … ┆ 160 ┆ 1089 ┆ 5 ┆ 40 │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 544 ┆ … ┆ 183 ┆ 1576 ┆ 5 ┆ 45 │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 554 ┆ … ┆ 116 ┆ 762 ┆ 6 ┆ 0 │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 213 ┆ 14 ┆ 55 │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 198 ┆ 22 ┆ 0 │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 764 ┆ 12 ┆ 10 │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 419 ┆ 11 ┆ 59 │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 431 ┆ 8 ┆ 40 │\n", + "└──────┴───────┴─────┴──────────┴───┴──────────┴──────────┴──────┴────────┘" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights.select(pl.col(pl.Int64))" ] @@ -796,10 +1757,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "id": "62f578d1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (336_776, 14)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delayflightair_timedistancehourminute
i64i64i64i64i64i64i64i64i64i64i64i64i64i64
20131151751528308191115452271400515
20131153352948508302017142271416529
20131154254029238503311411601089540
201311544545-110041022-187251831576545
201311554600-6812837-2546111676260
……………………………………
2013930null1455nullnull1634null3393null2131455
2013930null2200nullnull2312null3525null198220
2013930null1210nullnull1330null3461null7641210
2013930null1159nullnull1344null3572null4191159
2013930null840nullnull1020null3531null431840
" + ], + "text/plain": [ + "shape: (336_776, 14)\n", + "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────────┬──────┬────────┐\n", + "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ air_time ┆ distance ┆ hour ┆ minute │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n", + "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════════╪══════╪════════╡\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 517 ┆ … ┆ 227 ┆ 1400 ┆ 5 ┆ 15 │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 533 ┆ … ┆ 227 ┆ 1416 ┆ 5 ┆ 29 │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 542 ┆ … ┆ 160 ┆ 1089 ┆ 5 ┆ 40 │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 544 ┆ … ┆ 183 ┆ 1576 ┆ 5 ┆ 45 │\n", + "│ 2013 ┆ 1 ┆ 1 ┆ 554 ┆ … ┆ 116 ┆ 762 ┆ 6 ┆ 0 │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 213 ┆ 14 ┆ 55 │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 198 ┆ 22 ┆ 0 │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 764 ┆ 12 ┆ 10 │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 419 ┆ 11 ┆ 59 │\n", + "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 431 ┆ 8 ┆ 40 │\n", + "└──────┴───────┴─────┴──────────┴───┴──────────┴──────────┴──────┴────────┘" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import polars.selectors as S\n", "\n", @@ -812,10 +1811,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "id": "d4e486db", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (336_776, 2)
arr_timearr_delay
i64i64
83011
85020
92333
1004-18
812-25
……
nullnull
nullnull
nullnull
nullnull
nullnull
" + ], + "text/plain": [ + "shape: (336_776, 2)\n", + "┌──────────┬───────────┐\n", + "│ arr_time ┆ arr_delay │\n", + "│ --- ┆ --- │\n", + "│ i64 ┆ i64 │\n", + "╞══════════╪═══════════╡\n", + "│ 830 ┆ 11 │\n", + "│ 850 ┆ 20 │\n", + "│ 923 ┆ 33 │\n", + "│ 1004 ┆ -18 │\n", + "│ 812 ┆ -25 │\n", + "│ … ┆ … │\n", + "│ null ┆ null │\n", + "│ null ┆ null │\n", + "│ null ┆ null │\n", + "│ null ┆ null │\n", + "│ null ┆ null │\n", + "└──────────┴───────────┘" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Select columns that contain \"delay\" in their name\n", "flights.select(S.contains(\"delay\"))\n", @@ -844,10 +1881,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "id": "5e5c0dd0", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 10)
col0col1col2lettersnamescol5fruitnew_column0new_column1new_column2
i64i64i64i64i64i64stri64i32i32
012345"apple"056
67891011"orange"156
121314151617"pineapple"256
181920212223"mango"356
242526272829"kiwi"456
303132333435"lemon"556
" + ], + "text/plain": [ + "shape: (6, 10)\n", + "┌──────┬──────┬──────┬─────────┬───┬───────────┬─────────────┬─────────────┬─────────────┐\n", + "│ col0 ┆ col1 ┆ col2 ┆ letters ┆ … ┆ fruit ┆ new_column0 ┆ new_column1 ┆ new_column2 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ str ┆ i64 ┆ i32 ┆ i32 │\n", + "╞══════╪══════╪══════╪═════════╪═══╪═══════════╪═════════════╪═════════════╪═════════════╡\n", + "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ … ┆ apple ┆ 0 ┆ 5 ┆ 6 │\n", + "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ … ┆ orange ┆ 1 ┆ 5 ┆ 6 │\n", + "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ … ┆ pineapple ┆ 2 ┆ 5 ┆ 6 │\n", + "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ … ┆ mango ┆ 3 ┆ 5 ┆ 6 │\n", + "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ … ┆ kiwi ┆ 4 ┆ 5 ┆ 6 │\n", + "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ … ┆ lemon ┆ 5 ┆ 5 ┆ 6 │\n", + "└──────┴──────┴──────┴─────────┴───┴───────────┴─────────────┴─────────────┴─────────────┘" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.rename({\"col3\": \"letters\", \"col4\": \"names\", \"col6\": \"fruit\"})" ] @@ -862,10 +1932,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "id": "482d301f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 10)
COL0COL1COL2COL3COL4COL5COL6NEW_COLUMN0NEW_COLUMN1NEW_COLUMN2
i64i64i64i64i64i64stri64i32i32
012345"apple"056
67891011"orange"156
121314151617"pineapple"256
181920212223"mango"356
242526272829"kiwi"456
303132333435"lemon"556
" + ], + "text/plain": [ + "shape: (6, 10)\n", + "┌──────┬──────┬──────┬──────┬───┬───────────┬─────────────┬─────────────┬─────────────┐\n", + "│ COL0 ┆ COL1 ┆ COL2 ┆ COL3 ┆ … ┆ COL6 ┆ NEW_COLUMN0 ┆ NEW_COLUMN1 ┆ NEW_COLUMN2 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ str ┆ i64 ┆ i32 ┆ i32 │\n", + "╞══════╪══════╪══════╪══════╪═══╪═══════════╪═════════════╪═════════════╪═════════════╡\n", + "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ … ┆ apple ┆ 0 ┆ 5 ┆ 6 │\n", + "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ … ┆ orange ┆ 1 ┆ 5 ┆ 6 │\n", + "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ … ┆ pineapple ┆ 2 ┆ 5 ┆ 6 │\n", + "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ … ┆ mango ┆ 3 ┆ 5 ┆ 6 │\n", + "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ … ┆ kiwi ┆ 4 ┆ 5 ┆ 6 │\n", + "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ … ┆ lemon ┆ 5 ┆ 5 ┆ 6 │\n", + "└──────┴──────┴──────┴──────┴───┴───────────┴─────────────┴─────────────┴─────────────┘" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.rename(lambda column_name: column_name.upper())" ] @@ -902,10 +2005,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "id": "b7c0d519", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 6)
col0col1col2col3col4col5
i64i64i64i64i64i64
012345
67891011
121314151617
181920212223
242526272829
303132333435
" + ], + "text/plain": [ + "shape: (6, 6)\n", + "┌──────┬──────┬──────┬──────┬──────┬──────┐\n", + "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n", + "╞══════╪══════╪══════╪══════╪══════╪══════╡\n", + "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ 4 ┆ 5 │\n", + "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ 10 ┆ 11 │\n", + "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ 16 ┆ 17 │\n", + "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ 22 ┆ 23 │\n", + "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 │\n", + "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ 34 ┆ 35 │\n", + "└──────┴──────┴──────┴──────┴──────┴──────┘" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = pl.DataFrame(\n", " data=np.reshape(range(36), (6, 6)), schema=[\"col\" + str(i) for i in range(6)]\n", @@ -915,10 +2051,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "id": "3c2029cc", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 6)
col5col3col1col4col2col0
i64i64i64i64i64i64
531420
11971086
171513161412
232119222018
292725282624
353331343230
" + ], + "text/plain": [ + "shape: (6, 6)\n", + "┌──────┬──────┬──────┬──────┬──────┬──────┐\n", + "│ col5 ┆ col3 ┆ col1 ┆ col4 ┆ col2 ┆ col0 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n", + "╞══════╪══════╪══════╪══════╪══════╪══════╡\n", + "│ 5 ┆ 3 ┆ 1 ┆ 4 ┆ 2 ┆ 0 │\n", + "│ 11 ┆ 9 ┆ 7 ┆ 10 ┆ 8 ┆ 6 │\n", + "│ 17 ┆ 15 ┆ 13 ┆ 16 ┆ 14 ┆ 12 │\n", + "│ 23 ┆ 21 ┆ 19 ┆ 22 ┆ 20 ┆ 18 │\n", + "│ 29 ┆ 27 ┆ 25 ┆ 28 ┆ 26 ┆ 24 │\n", + "│ 35 ┆ 33 ┆ 31 ┆ 34 ┆ 32 ┆ 30 │\n", + "└──────┴──────┴──────┴──────┴──────┴──────┘" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = df.select([\"col5\", \"col3\", \"col1\", \"col4\", \"col2\", \"col0\"])\n", "df" @@ -934,10 +2103,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "id": "e7ab5f64", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 6)
col5col4col3col2col1col0
i64i64i64i64i64i64
543210
11109876
171615141312
232221201918
292827262524
353433323130
" + ], + "text/plain": [ + "shape: (6, 6)\n", + "┌──────┬──────┬──────┬──────┬──────┬──────┐\n", + "│ col5 ┆ col4 ┆ col3 ┆ col2 ┆ col1 ┆ col0 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n", + "╞══════╪══════╪══════╪══════╪══════╪══════╡\n", + "│ 5 ┆ 4 ┆ 3 ┆ 2 ┆ 1 ┆ 0 │\n", + "│ 11 ┆ 10 ┆ 9 ┆ 8 ┆ 7 ┆ 6 │\n", + "│ 17 ┆ 16 ┆ 15 ┆ 14 ┆ 13 ┆ 12 │\n", + "│ 23 ┆ 22 ┆ 21 ┆ 20 ┆ 19 ┆ 18 │\n", + "│ 29 ┆ 28 ┆ 27 ┆ 26 ┆ 25 ┆ 24 │\n", + "│ 35 ┆ 34 ┆ 33 ┆ 32 ┆ 31 ┆ 30 │\n", + "└──────┴──────┴──────┴──────┴──────┴──────┘" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Alphabetical order\n", "df.select(sorted(df.columns))\n", @@ -982,10 +2184,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "id": "62e540c0", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (12, 2)
monthdep_delay
i64f64
721.727787
620.846332
512.986859
1216.576688
210.816843
……
812.61104
110.036665
313.227076
96.722476
106.243988
" + ], + "text/plain": [ + "shape: (12, 2)\n", + "┌───────┬───────────┐\n", + "│ month ┆ dep_delay │\n", + "│ --- ┆ --- │\n", + "│ i64 ┆ f64 │\n", + "╞═══════╪═══════════╡\n", + "│ 7 ┆ 21.727787 │\n", + "│ 6 ┆ 20.846332 │\n", + "│ 5 ┆ 12.986859 │\n", + "│ 12 ┆ 16.576688 │\n", + "│ 2 ┆ 10.816843 │\n", + "│ … ┆ … │\n", + "│ 8 ┆ 12.61104 │\n", + "│ 1 ┆ 10.036665 │\n", + "│ 3 ┆ 13.227076 │\n", + "│ 9 ┆ 6.722476 │\n", + "│ 10 ┆ 6.243988 │\n", + "└───────┴───────────┘" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights.group_by(\"month\").agg(pl.col(\"dep_delay\").mean())" ] @@ -1023,10 +2263,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "id": "af588177", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (12, 3)
monthmean_delaycount_flights
i64f64u32
313.22707627973
1216.57668827110
620.84633227234
96.72247627122
110.03666526483
………
721.72778728485
210.81684323690
812.6110428841
512.98685928233
115.43536227035
" + ], + "text/plain": [ + "shape: (12, 3)\n", + "┌───────┬────────────┬───────────────┐\n", + "│ month ┆ mean_delay ┆ count_flights │\n", + "│ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ f64 ┆ u32 │\n", + "╞═══════╪════════════╪═══════════════╡\n", + "│ 3 ┆ 13.227076 ┆ 27973 │\n", + "│ 12 ┆ 16.576688 ┆ 27110 │\n", + "│ 6 ┆ 20.846332 ┆ 27234 │\n", + "│ 9 ┆ 6.722476 ┆ 27122 │\n", + "│ 1 ┆ 10.036665 ┆ 26483 │\n", + "│ … ┆ … ┆ … │\n", + "│ 7 ┆ 21.727787 ┆ 28485 │\n", + "│ 2 ┆ 10.816843 ┆ 23690 │\n", + "│ 8 ┆ 12.61104 ┆ 28841 │\n", + "│ 5 ┆ 12.986859 ┆ 28233 │\n", + "│ 11 ┆ 5.435362 ┆ 27035 │\n", + "└───────┴────────────┴───────────────┘" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Multiple aggregations using polars' syntactic sugar (shorthand) for mean and count\n", "flights.group_by(\"month\").agg(\n", @@ -1055,10 +2333,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "id": "b0e56ff1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (12, 4)
monthyearmean_delaycount_flights
i64i64f64u32
2201310.81684323690
6201320.84633227234
7201321.72778728485
1020136.24398828653
1201310.03666526483
…………
920136.72247627122
3201313.22707627973
1120135.43536227035
12201316.57668827110
4201313.93803827662
" + ], + "text/plain": [ + "shape: (12, 4)\n", + "┌───────┬──────┬────────────┬───────────────┐\n", + "│ month ┆ year ┆ mean_delay ┆ count_flights │\n", + "│ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ i64 ┆ f64 ┆ u32 │\n", + "╞═══════╪══════╪════════════╪═══════════════╡\n", + "│ 2 ┆ 2013 ┆ 10.816843 ┆ 23690 │\n", + "│ 6 ┆ 2013 ┆ 20.846332 ┆ 27234 │\n", + "│ 7 ┆ 2013 ┆ 21.727787 ┆ 28485 │\n", + "│ 10 ┆ 2013 ┆ 6.243988 ┆ 28653 │\n", + "│ 1 ┆ 2013 ┆ 10.036665 ┆ 26483 │\n", + "│ … ┆ … ┆ … ┆ … │\n", + "│ 9 ┆ 2013 ┆ 6.722476 ┆ 27122 │\n", + "│ 3 ┆ 2013 ┆ 13.227076 ┆ 27973 │\n", + "│ 11 ┆ 2013 ┆ 5.435362 ┆ 27035 │\n", + "│ 12 ┆ 2013 ┆ 16.576688 ┆ 27110 │\n", + "│ 4 ┆ 2013 ┆ 13.938038 ┆ 27662 │\n", + "└───────┴──────┴────────────┴───────────────┘" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "month_year_delay = flights.group_by(\"month\", \"year\").agg(\n", " mean_delay=pl.mean(\"dep_delay\"),\n", @@ -1104,7 +2420,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/data-visualise.ipynb b/data-visualise.ipynb index 488a1db..0e2feeb 100644 --- a/data-visualise.ipynb +++ b/data-visualise.ipynb @@ -62,10 +62,51 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "a86fb211", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "import polars as pl\n", "from lets_plot import *\n", @@ -122,10 +163,49 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "0cf986aa", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (344, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64f64f64stri64
"Adelie""Torgersen"39.118.7181.03750.0"male"2007
"Adelie""Torgersen"39.517.4186.03800.0"female"2007
"Adelie""Torgersen"40.318.0195.03250.0"female"2007
"Adelie""Torgersen"nullnullnullnullnull2007
"Adelie""Torgersen"36.719.3193.03450.0"female"2007
……………………
"Chinstrap""Dream"55.819.8207.04000.0"male"2009
"Chinstrap""Dream"43.518.1202.03400.0"female"2009
"Chinstrap""Dream"49.618.2193.03775.0"male"2009
"Chinstrap""Dream"50.819.0210.04100.0"male"2009
"Chinstrap""Dream"50.218.7198.03775.0"female"2009
" + ], + "text/plain": [ + "shape: (344, 8)\n", + "┌───────────┬───────────┬──────────────┬──────────────┬──────────────┬─────────────┬────────┬──────┐\n", + "│ species ┆ island ┆ bill_length_ ┆ bill_depth_m ┆ flipper_leng ┆ body_mass_g ┆ sex ┆ year │\n", + "│ --- ┆ --- ┆ mm ┆ m ┆ th_mm ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ --- ┆ --- ┆ --- ┆ f64 ┆ str ┆ i64 │\n", + "│ ┆ ┆ f64 ┆ f64 ┆ f64 ┆ ┆ ┆ │\n", + "╞═══════════╪═══════════╪══════════════╪══════════════╪══════════════╪═════════════╪════════╪══════╡\n", + "│ Adelie ┆ Torgersen ┆ 39.1 ┆ 18.7 ┆ 181.0 ┆ 3750.0 ┆ male ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 39.5 ┆ 17.4 ┆ 186.0 ┆ 3800.0 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 40.3 ┆ 18.0 ┆ 195.0 ┆ 3250.0 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ null ┆ null ┆ null ┆ null ┆ null ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 36.7 ┆ 19.3 ┆ 193.0 ┆ 3450.0 ┆ female ┆ 2007 │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ Chinstrap ┆ Dream ┆ 55.8 ┆ 19.8 ┆ 207.0 ┆ 4000.0 ┆ male ┆ 2009 │\n", + "│ Chinstrap ┆ Dream ┆ 43.5 ┆ 18.1 ┆ 202.0 ┆ 3400.0 ┆ female ┆ 2009 │\n", + "│ Chinstrap ┆ Dream ┆ 49.6 ┆ 18.2 ┆ 193.0 ┆ 3775.0 ┆ male ┆ 2009 │\n", + "│ Chinstrap ┆ Dream ┆ 50.8 ┆ 19.0 ┆ 210.0 ┆ 4100.0 ┆ male ┆ 2009 │\n", + "│ Chinstrap ┆ Dream ┆ 50.2 ┆ 18.7 ┆ 198.0 ┆ 3775.0 ┆ female ┆ 2009 │\n", + "└───────────┴───────────┴──────────────┴──────────────┴──────────────┴─────────────┴────────┴──────┘" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "penguins = pl.from_pandas(load_penguins())\n", "penguins" @@ -141,10 +221,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "23c75ba7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (5, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64f64f64stri64
"Adelie""Torgersen"39.118.7181.03750.0"male"2007
"Adelie""Torgersen"39.517.4186.03800.0"female"2007
"Adelie""Torgersen"40.318.0195.03250.0"female"2007
"Adelie""Torgersen"nullnullnullnullnull2007
"Adelie""Torgersen"36.719.3193.03450.0"female"2007
" + ], + "text/plain": [ + "shape: (5, 8)\n", + "┌─────────┬───────────┬───────────────┬───────────────┬──────────────┬─────────────┬────────┬──────┐\n", + "│ species ┆ island ┆ bill_length_m ┆ bill_depth_mm ┆ flipper_leng ┆ body_mass_g ┆ sex ┆ year │\n", + "│ --- ┆ --- ┆ m ┆ --- ┆ th_mm ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ --- ┆ f64 ┆ --- ┆ f64 ┆ str ┆ i64 │\n", + "│ ┆ ┆ f64 ┆ ┆ f64 ┆ ┆ ┆ │\n", + "╞═════════╪═══════════╪═══════════════╪═══════════════╪══════════════╪═════════════╪════════╪══════╡\n", + "│ Adelie ┆ Torgersen ┆ 39.1 ┆ 18.7 ┆ 181.0 ┆ 3750.0 ┆ male ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 39.5 ┆ 17.4 ┆ 186.0 ┆ 3800.0 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 40.3 ┆ 18.0 ┆ 195.0 ┆ 3250.0 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ null ┆ null ┆ null ┆ null ┆ null ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 36.7 ┆ 19.3 ┆ 193.0 ┆ 3450.0 ┆ female ┆ 2007 │\n", + "└─────────┴───────────┴───────────────┴───────────────┴──────────────┴─────────────┴────────┴──────┘" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "penguins.head()" ] @@ -177,14 +290,158 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "574fe39f", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(penguins, aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -247,10 +504,120 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "15c3848b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(data=penguins, mapping=aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -291,10 +658,122 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "6b0e1c38", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(\n", @@ -325,10 +804,136 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "943efd36", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(\n", @@ -356,10 +961,135 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "9e12b3bf", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(data=penguins, mapping=aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -382,10 +1112,136 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "17d5803b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(data=penguins, mapping=aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -406,10 +1262,154 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "b9b98ec4", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(data=penguins, mapping=aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -470,14 +1470,138 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "7c76be4b", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(data=penguins, mapping=aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -590,10 +1714,119 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "21b45061", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(penguins, aes(x=\"species\")) + geom_bar())" ] @@ -610,10 +1843,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "4e046bb2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (5, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
catstrf64f64f64f64stri64
"Adelie""Torgersen"39.118.7181.03750.0"male"2007
"Adelie""Torgersen"39.517.4186.03800.0"female"2007
"Adelie""Torgersen"40.318.0195.03250.0"female"2007
"Adelie""Torgersen"nullnullnullnullnull2007
"Adelie""Torgersen"36.719.3193.03450.0"female"2007
" + ], + "text/plain": [ + "shape: (5, 8)\n", + "┌─────────┬───────────┬───────────────┬───────────────┬──────────────┬─────────────┬────────┬──────┐\n", + "│ species ┆ island ┆ bill_length_m ┆ bill_depth_mm ┆ flipper_leng ┆ body_mass_g ┆ sex ┆ year │\n", + "│ --- ┆ --- ┆ m ┆ --- ┆ th_mm ┆ --- ┆ --- ┆ --- │\n", + "│ cat ┆ str ┆ --- ┆ f64 ┆ --- ┆ f64 ┆ str ┆ i64 │\n", + "│ ┆ ┆ f64 ┆ ┆ f64 ┆ ┆ ┆ │\n", + "╞═════════╪═══════════╪═══════════════╪═══════════════╪══════════════╪═════════════╪════════╪══════╡\n", + "│ Adelie ┆ Torgersen ┆ 39.1 ┆ 18.7 ┆ 181.0 ┆ 3750.0 ┆ male ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 39.5 ┆ 17.4 ┆ 186.0 ┆ 3800.0 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 40.3 ┆ 18.0 ┆ 195.0 ┆ 3250.0 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ null ┆ null ┆ null ┆ null ┆ null ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 36.7 ┆ 19.3 ┆ 193.0 ┆ 3450.0 ┆ female ┆ 2007 │\n", + "└─────────┴───────────┴───────────────┴───────────────┴──────────────┴─────────────┴────────┴──────┘" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "penguins = penguins.cast({\"species\": pl.Categorical})\n", "penguins.head()" @@ -641,10 +1907,120 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "93675336", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(penguins, aes(x=\"body_mass_g\")) + geom_histogram(binwidth=200))" ] @@ -680,10 +2056,120 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "6a58021f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(penguins, aes(x=\"body_mass_g\")) + geom_density())" ] @@ -752,10 +2238,141 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "a636947a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(penguins, aes(x=\"species\", y=\"body_mass_g\")) + geom_boxplot())" ] @@ -770,10 +2387,123 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "9b85a2df", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(penguins, aes(x=\"body_mass_g\", color=\"species\")) + geom_density(size=2))" ] @@ -792,10 +2522,124 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "353189e5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(penguins, aes(x=\"body_mass_g\", color=\"species\", fill=\"species\"))\n", @@ -833,10 +2677,121 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "e091e211", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(penguins, aes(x=\"island\", fill=\"species\")) + geom_bar())" ] @@ -853,10 +2808,122 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "7df8fb7a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(penguins, aes(x=\"island\", fill=\"species\")) + geom_bar(position=\"fill\"))" ] @@ -882,10 +2949,120 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "5066527d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(penguins, aes(x=\"flipper_length_mm\", y=\"body_mass_g\")) + geom_point())" ] @@ -904,10 +3081,124 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "8ca23d34", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(penguins, aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -931,10 +3222,130 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "00dd36e3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(penguins, aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -1004,10 +3415,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "3410634b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'/Users/omagic/Documents/GitHub/python4DSpolars/lets-plot-images/penguin-plot.svg'" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "plotted_data = (\n", " ggplot(penguins, aes(x=\"flipper_length_mm\", y=\"body_mass_g\")) + geom_point()\n", @@ -1029,7 +3451,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "id": "852afe51", "metadata": { "tags": [ @@ -1119,7 +3541,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/data/bake_sale.xlsx b/data/bake_sale.xlsx index e122900cc12c6d5ee3e5bbbecc52b1a202f53865..5de6f9a30425e19aa46b807093c289a907c83332 100644 GIT binary patch literal 6192 zcmZ`-1yoe+79F}fBnD6rq+3D+>Fy3`#$gy3I;5n#1(jw95fBulq(ea%NQ!g1ONcIsOtg1m}0Wu3=IGv!2$p%Q2&_8 zIm0|aFb_-p$1Wgub6y`O$MV!cSO-607t$|=bG!s^i(i8xyz>$$T3CsL8?DANd^-js zv@UlFeez|zV0)1v^z1A^vpkM1bMg^1%D#!~YZ8euw&r7}>3l2XyVdgq(2K&6VP71q zDfFtI69?fb?6$ZJ2I|+U9Mc5Br6H!w$7n#1Dibt)?Jlo;k^{$$n=tH9GkBBQylxI1 zkqD!tEQgT@NQ?e-nl{=h8AM`k-O2fau$_*T;@ii`b*5B*rTNi`F)I+dg&rD3D|trW zT#)%3SGKw^T~$fky26+~YqinH8U}vpYPUgP)E7yF>wkHw)EG2Uc%;|Rq`35mb@vhGD@ku&T zJm!g)2%e)Zu-?Kw)AL?-J9UQW;K!rLtO%nOIH&f|s=48K zvpp9az9Ty15BQT2 zc~7&QH53!?QH;=`Rv$;cU!-_DyFqQ7ouNPL^FIVRbn=tRBQNkPk4?SERXLK~Il>=K z9bkl2*Qz_J*w;ZWj>+&e7;xb@xgX-brsFhj_74ulVz6z57->_n;vTSl9jL>X@^(%f zyctzf*FeHZ6{q~RglP=<6*?fa+@Q=H>7zUUVktJ&4oI94L;86&m#t97{aHzxPG&Pq zhqrf6x~1F1^(ib=5BjkKia%D|T5NEU>61!Mehfd{;7smH9u)LGBD<)cU@bRR0KfL zHjYufZwb*%fFUd!Sv_NZail4eyydntmi}Z_R45zbd-%D=k)(@rl$OK@4V*{7omL@q zdFtjJ*pu$5`@Bkli1rE-@n-McLD0&K2q1w>dMZwI;4qrB);{m4SGc6TE~jv|hh2v0 z*V!QM$h-9fx(v4&`d1(o~b@3ZKgCn}hD`(c&rE!*# zJsN!qImGF6l)$a6zN~2}IfC%el+)Ed(OL!n<3)`3=uDiNHpqzl96~O5O5@l?cchb99Y24~FV*ahU9u&-B%}3}f!H*;nHp z&fZ#}WMqJM(|?X4em1eGaC92hEM@Zsywxvw&qBWY&e{ns!{E?!C3^wCJ&Um&c8&6C z5Rd)`0>djYp{1~hcb$P^PGVE}?q4sAiJUlSoOM{J^3Me9ORIT!?yD?ubGZ~>RR8Eh zsOQ^*`{Zbeq5FQ>aqEa>%RjRiO5?z)jLIS#s>(@zXYoJv9j}8xz5Aq1`&47I5Jm?U z)4L@f)bwS8v%-c95>*Vtmy(?!vwwGiSSCBy+tF?0Xd~W`M#oV=|Sm3p>7xlP)=( zbk?oL^fCUO9=Hn zxp6}qWMBk3s5^VK<3PMnuVD*f*F$LhU_P6&h3!`C>W%Ms-&&5u``&RAuYBe^UI?@W z`Qjcky2xHpZe$$YalDZdWFBkhLes5Up{h#JJM=;m+ygZJ!r`ejSr#bb&yOo-o3cp% zC+VjLCZ7pWq<=wOKaWF{w0KzCID*`NEq}c>pv0b^q7s1&z!x}H)MPIT`wAzBkY0yF zn5-NUZG)+egkXzxZ?Zh1dsNoOTm4PTNd`*!VjM#Tt)&ps$!}zx&Uqt`>-J8GDG5&L zoOV(>r3p6|9avASRf^uO0!@=)C3lT@#FtIhJ{3P_laEZPa4mix8+Nx&e(|-1-9)*U z{e5=5VgqePb51_nd*m`s_3y-*)ITU|5FhV<>m*{w3wdVQ88{UYusD!sY|RH3He}B+ z^K_LBAguMyF`)A06X&+d*Aj?JQX^$Q@vs*;G=9VPJ=XFJoM;h8`1F&1+}i$nZbUiZ zGm#d&7z#fKGopuo09XTVO6tm06=v8ab5GSwcyRY@j;jUjPxf#`R<){7tdaa?&)weI z4P>Y1;Rb<$-G7P>C{YH6E(n2Wx4==RUa+T-+Q0_Rd17s?*L7Z zp!b8GWEFWsc$46;@#1H}FlKpSy(G@QFJ=W?<9SM#PtTd#$ z3D{%8XrNg^WcvmWn_ZN5K7*sFoC!Sw@)-ayxy)2-NH*}G@Em6E+wwsUd{)RtX_(bm_bJ@k@O6LHs=l6EX zr#p1J#;|H3Kr=#wej+NxWs2aGl=9n${uYX1f{RGkTX!COc?az;u=DTguCd(y6iS!# zefL;UD4|wt^!3v~Q5f|B`4cqWdxx;sr<@60BnHn<-0M%vG$@;9L3d^gcZoRX($+2>MhsY{@c=q7nvvw|6W@*Q~jztTC%d3 z*6($C56oTeI~g9H4(W~CFd(URNm_lXL3L#346M^*i`v49%!+c~@Yflp2t zRyS&J1mu+Dm3yDoetTG95_MaO#GbsiTq5+c=vjDShe{Bwmhw~vImCSGbe?lopDY>= zeNP^?SWCr`2Aixsu%<^K?z7@e)4P({VyTf$2Df#$i9GUZHiS+Jyo?Sf2V#s*6_6d` z(b;jAm(%VuhQ5*SaqI*y4h^zyP>PGtNK7PV_3~1z;l8WIazhWif0HCNoaPxmth(Iv zvU01u>?kX$gsPg({J~YK2dfN!#i|E0Z~!m6rO_1x9%!}LvFtkZZH5h|?<3H@9=Gd) z@ivdlG{IeZ7*x*@O^tWotQhXnU3HNM7L2U>&{O%_275o>DK{ONsW95vMo02&&P{B4 zWl}ZOwTV98x!|JLsEZktAuH(d!OQ;W&X_K4_hEvFr3q_ij2n%b-3(C?BBfL2#?y8D zehuR$39q5Bj6+5W*%8|K$6amhEWEZ%OsIDZk0q}Ys4D3IOT%F5l>MVN8NsKr)dx+! zmz2%Em&?lw@190J$i46DvA1&vV}U3u?A`(bDRs4(sWp_V)f({QL`}$I<76)I{9xR& zg&!l0*njuS)45#9$GeB%C;nGo0dv$ouR3vr*~n*@yVructukc*v@!cSgv{z(lw#3HL}3|42Th5+hqn@`WE%t=@TbVdQw@r(E_iSA-X}~iEWyH- zMqdeOFf*LNfpRf6)q`m|coyYJjzq3)J-se-Me^BiUL%M21@9<(fthF4X5RHwkE@oLa zxhXU1bP<0^P0kc_?k7(rK7E_c=a&CNJx9^SB&KX$Hh3R;yZ-b==^B3rN0h4@9-P}G zlDz0)hf14ikHiB_U4E&#{YPMFY1^9aP@6`0F43yvo&me3Onh8>LkFGFtmX?nzi+QWg1s3J-^TwlUi5z8PDdbDxdWX&+j*?%*J3#} zMV7@fDv4Oo<3ue=vPd15xClpPV*a)7)Du+Nc!P1Lj<@n;X9Rak(<}|!*Nucho>tEk z>$g;Rc#g2{x8`gny%9d#qH1V`bHoz}fAsU#=n=fpHKv%v^va5mlm;X|SzE777Rj5c zIEo~o0lJyw-3Y1rq%X_uH;Hm@W6o?jP7SUyCySTFgeon@L=OGLWfcf8P~|7w{ik5z1^3ozO6cK;Z3$87Psm3yor|^bP*R4>NvK1n_xqSD z#tv$f+PkJoTn3b!W*l8-%+7Q&I~FiNmP7R=I} zgO&Il{G{0PGsU=dnbpM1!_~34=0#F%=?_P2GQ)1i1o&axV;2*fy*Qt9HQ0Y~$E%oB z3Cx<>1W+^-mo#Nrm}VH4uo8fO5ffJ5B`PqEo}E7@%w3@zK6?($Wx66no(iT zqXnbdwNu)DK=+ANb7V+8SzB_-a{t_M)j2~wD{8mes7ZtjwPQPHTWvRI7k55eXE)H# zwkJCU@v|dVIKIFyXy%!MXpq)oIQ5hU%bJ@uPz>KDt6@}j$UYCCYE6+gG*8&h7V^eU zD+_fjuzag&Zvf|3(ZP&?@Qzl-*qt*7cPhOZ=JTrNv zyH+wth>%BFdv<}lXJ=xwUDf*?+=w?z_oN^=b4(>`2ItNr+i_2oy0!9KPa+#m9}d@w^h-=#UAcV1$aW7=WIyeT)W^27MPq$pH> z6=LyWJs^uOllx9=$TEp#ak9|`I1dEAiE zc5(SdX}jj;PfAhNj-;pVtpj~=#aBVbJMu-;#`NG}W3k$AwE558bgO(xLg%V#5OVhn z5Uj1AtP$C~nZH|M_mLY+EB2jhsg~7mayt9UmNvyLt!h@L=O}1PH7DBO0F9Zpjhj`B z+eD6%I;#MdXCS6#`<$&Nu{>YDi|5$WNqE}lVm5?DYH(kTSOVQh#*?v2+DrMN$HDke z?`953uIO3PUSU0PJ6$>ZMd7)#$H5x37v@d(dVU11ECU@vejc(|3DHuk9P8~6#6p;w zV79l?Ei&r2zDpA%D_XPv9L&ZfB+LYXx<6~?3}Y;|%Z^bLx51f$(^L~F>$IGlL=XYs z^S16udPx0D0u#cB+nVwIm@21~uC|}hrN+3Z)^9A@sNUy)i?`IU$`!c7pmqAJ`M^n= zmnSK4qrG0#AD{k70dB(PeWw`D0%rYn>@LqlBG=sd{O=bC#rHh=N`r$kPkszW;_$}) z;MSURS?$i5w{$iML|JcW=#&`$-v~uz>aR~w@XzHRY|-lg*WJ5c;Q&BTa2zW8fAjLL z2VOV1eh22DUc>)ydHoO0H8bcpP7A8^P&j`RuivtD9p$U*!CNN9`6^ze}}m1q1QY5-=PvHKNj`y z>;3(8fa@p5ZvY@lPXA27Kb{}g0j~@4Z$M)5e~{@H`g>o0^8fy~>$B(DuAO_=bD#6MpL3rX>f+$i0000&%%u##eNRbcg9QLk-~j;C zm`|+LJU#qi9)7kaL0&Ll8__^_x9W^Bk8W|&K6LsK(De09rdj)3UMt|a@28YLR(5XD zM6GmwAn9VL2Z{4DdRT;R^~}-nd7c$RBBD{-jKWaiS-riq9g)R)M$>*?h@S1o;4C{D zGolsYRX3aueeVyaom+Pw5&2sWThhJ=`@#&vYkW5x^d>{Vrd*>0A&$EJA^Is!CkyF! zxSw=HqKg~Y0`wrlSXf`L0CCXbOZARjm-TITOs3>g`-HLj;C`lQC;xg@ijk9bWad$z z8eRe$H#nyg3)vwfpIb!BO-;F6Szhs2h>x!;Xw{^U#azyAGV^7Y3gb|&m4q{?fMErn zHlb0^;=WOnAJzr==X|5d5KeD**}g<9#bA>Itf+2c_8Hzy_23VwM?|ft9k)=SNt_>k z_8?ym?{)iUPp2VB>JxX+o^+n_p#fL$&yGks$uk8TctfmNgXdR4N|pA0mE-aEGU&HQ zqXeGl`ehQszp`QY5ZbkY$$}m>06>CSZGB*FzG9-6uhnTIIyfMTN9YJN#oFRXQ$!pX zsZ05Qx`JGsw_bURmpaX;+0&DPMyT=N3n)FuvxC90J?Cf1BLCO$$Z(ixp~7`R@I-_GDIg-aDl`P&hjTu zx)sgvh|d~^Hhi)>QBwuDEX1xu@z+3^8{BGho6SiM=N8?kk}G-RBVnSkj>TU-9ck*l$(>Jbmun@$|fRS)%`8sLQoSoI)KPeGxeu zETSOtGUG!_6y&x085YFKnzxC&0a7&RVga7S}%ICA0ZM)0@ytknKpLgkOC za34^+p{(%=kJWtN-yP-kV? zufETW^|o#4gS@>D$^T|ZA@cZFD&`grzySd0e=~H+jjuBd=I1N+&jPw6DKTBwph}$P z(Xq^~GI&7&FJ_k!n?4cLDh}#6Vv$whI!I}f^hI|b)6fi%k_4b8_Vb-4E5mGtj_j%+ z`MX}UUihu=jMwE?JZl0_*?E?H{sPuhDZ9VyPEA9P_^ub7z9D<$pgu0WlIR=4870E_ zt;EKjJXeu67bD~Zw6`;@!1kz}^)Ff=1dZ$5I7Dh5 zW}#dvKA@QH;J-?+jUCV*)+hkxe5PgI(sP7**cL^1t7_wX8J20%_^~reAc3c3lGnRP zrg5FY48jvo9o*M*y}Z>McTR84<7Ik~>(0AjKy&*+J#l*6hH%?%ZWYbS<|0Z}@F=6| zWN%?OWNDSU_gT>d=ahA9)XhM@C{|l_E_n;b8oIen>Z7x`_JUfOp>KhdEQLiYTz-mb zwyeeP-B#PHmW*<(T!yU5s48$m)7RQ>63UC*-y|8e?A5Bp}LDi6=sY8cbWW zt`(GcXVJnZLy-?1VZ26ul554JvGu1x(7^-Pp26Q~G#>tj*2Sce4^!ymf71BBC7zh7 zZ_p!78iUsQ5&I!vhTNU@*$i}9R}b>2cXqVc8{zC?%p<_^i?!?~!6(_d8VBXQO=0tX z&02#7C}X~wM0|L$XkyBP!k+WBr47cX`?pA5C-gCQjCZ|)w^FMV}4KA?1qaq0rM zwj5q|Xd_}x@V+jMxS8)pLi4tyr!)Emb}4-S81jHdPER|R^%spBV$shS5A?)|g4QX$ zCX1*luS8PmF7jbCo5-hF_2I4f90ArF#R0?#x!S;oY@i&AQzACuT^5kKu5ur2+A4`;5jaiLX-IdW z#PjbKeehI+{Sb{!KHghnr6D;ZcHhZRGawyWK6B-O*OnO1`vMcS6cD-1WOA$8&I4uH z^6If^uf5A!5~FBk)R(IYU~~ERofY|SzO*!CwQ&`5kf(yc;El7LNxfSBK=R4c6%l@- zh^)`up$kzV*keT&c6`NPGyVc=e{V<#ag%Ssw6x!CpQl|~Y!37s+3bcpr-n7Jif`M_ zJ9Qq70o&=-y=82&Vlwb~=4&S$VH2sw~8 z?1#8s00aZ0(5~JF1KopQV^ghi;m_y4#5#~6*#mjCI*xlg@b!1>Jl`c*u0L!;BgE{~ zxemSWK*;H6W}~Mgsg3I0v=vET?N>IY;R_o{D{_ep@FA%DyB;)1m?yN8##!;6Px+5E z_3GJC$rzJcizn_?EFr}R_~c~CENf*eyXJ&Gf2z>+fw6pwsJ)FA)lNs2pyLj!vo7@HIt%HqilS6)B zU$QCcvkekT}EKVZUGh(UEOS{bZy%eFxVr zn`TFI61Bp}>)dMTxsvGn$Y~F&sg>TG*~x_KGsAbATZ)h3qN4cYNT_6hgymB@w+3r0 z`6D~O?rp2)rC4c4hur4+pD0SCmYpQMgKuy)t4x*zp zdqWd`o$7?}M05Bx!c||P8{A}=z-1SFPq9=m}}^GS|+K4(u7 zFZ7Jdscr~YzPHLfJdaYDG39&^LaK`=GjAqxZgDgN3s&tl!K>Cpj49#SsEX%KNy<}_ z(nUc(FTfk>zu1nQ)+|40zuk-yK7j$Y&h0AdBmB=r$|O-0;d?`-iwn8tzC(eDVGekm z6YG9$iBui6y>Eq1%&ONt5W0Z@@yBL|Xt#j3BUFpl%g=;PE?y>rA!3YML<_t z;)by$*0ypf7qJ8H6##TIs$u3(w#rwxp6px~Jp+7*ctV?)n#*%HA>^IEH<*T|410&S zYd-(sP_~eDg)n!BD)n@e5>(uzmuhcsE?KPbx+>;U3f(z4=@RQFXJb`9nNtZQ>ko&X zET4;ATpZmzK%}E@E1nG|WXoS17b=}^Bg95jUg8vn{X9u7OD+uCKP^;Nlz7*S$GWkD zl^CZkGe9|yqMUW4P>ES~qlu)Lq{0JYxd4t#EU*{z>>a!PwdR5XaKQlTosEOr+K5O) zJz6P1O{@_Sd`NE7%N~q+AgC+j#7N+{bm!H|5l9I<@V$W0-7{B(_a^m_hn9}(cTk)( zP)m*b;KbSt7G{;M%R|b2jNk7RAJ4 z!qv0$j`I$5$CHreS6PVdrXl=$o4($CVy~Ff`ZH>Aq#RabhOO#xU)~6r9lpjeafjCc z0vGPh^a_7BCq;ED1Y#OUv5ea*n-isStqLceN_2_a&U9}yDV-)^?5OROfnwZZd(Wt2 zVSFrxM0!pzrL{^K1Bh`MJmY7*IemMw#2vh-H!rXH@qG*Ja+`0Qu{)2>k*i!CybW@r zDAW8{cy*=3J<&cbVu&9rRKi{5DV{fS&(wZU=*ekG*II+0YbJiSIydAMi*Qe#;9IFM ze=4!VnqZHO*nQBu@x+j$yG$U#XO+wEsVOc~MQKw}J5e`|(+t2}KhX|ok zWm?!x&cP}Xkg}V_>~{@l?{`Gx+(Z=C_-bktT9yl?{|L|m5;2ubBlWhM4Q0{1(OeRr z28!lNHyH7MHV5u`3-K0C_D8;CKNhZ!tCR}aHhQz`D44fvWG!QT>v$$~ir4EzWt|S} zA^5N<&y(OG+wn8Ns}eRyZP{ z$!VuCptW*)nyRreQm@JCbS0h~9lpgP%del4c8V-H_nO)o{UrfVI!g1~ykbVn^9}*^ zn8Tk3(+F_=ZUX+xYW?=5e+<_@t~7D1mbhD-0{@H64omkoZ*^oGc$*VqUP10vijbb= z<%qxYdGQhO8Z$FG^a2fk{s77qoI*a&NF+D*DAlNGx-{F%1^Vpz8;XerSt~UvM`FXw zO;hhuN(aGc)wl1`?nFd!Xa-PlmWT6_ev+3ESg$G+FP%8jSYD}-^{OJLn$?085$RdNTrJFDB};`KVG9=RSahzga{W1T{+gksg&h> znEe`Idg6FBu-St za>F#S;8djAle3H0cVooemquK1W=On@*adDptr#B-RujMPMv&IZEo(v{Gq1Cu=F&WC zc%jOhwvdTv%A6_dX_r47dg4W3muyz8uYR<+~16$_M-xjlf@Bl2<1Z9Mko5@y`sTqFBGcaOQEM_D8Nt`3!CO2EBNmt zQcSk~c{GJzF8?wxy$W!3s`on_0B8zN!c@S2&G@bcUY&jY2`t9kjQ{^h*j1dX6QVyj zJ(%*t;QW_}{gJP$C|3s*e^4G{y59ew{O`EpYVcK;{U;ck;NLy>RfMZP^A7?9lPAp2 z|39wtD#BF@_Xk0M=r_Xu8M>>XS9|+Ep}LqkE#}9scKTNVuAUr!07NmO`gaQc^&Ghh xcvY7F0CH0Oi%4G$zp6if!UHjX1N@)H|3{4sb@A{ozd?$5Qe#SngZA>>{{X;;djbFe diff --git a/databases.ipynb b/databases.ipynb index 661cfd1..3c786b9 100644 --- a/databases.ipynb +++ b/databases.ipynb @@ -22,7 +22,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "9e54fbf6", "metadata": {}, "outputs": [], @@ -92,10 +92,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "970d2c19", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[(1, 'AC/DC'),\n", + " (2, 'Accept'),\n", + " (3, 'Aerosmith'),\n", + " (4, 'Alanis Morissette'),\n", + " (5, 'Alice In Chains'),\n", + " (6, 'Antônio Carlos Jobim'),\n", + " (7, 'Apocalyptica'),\n", + " (8, 'Audioslave'),\n", + " (9, 'BackBeat'),\n", + " (10, 'Billy Cobham')]" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import sqlite3\n", "\n", @@ -116,10 +136,47 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "c5871b6e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (10, 2)
column_0column_1
i64str
1"AC/DC"
2"Accept"
3"Aerosmith"
4"Alanis Morissette"
5"Alice In Chains"
6"Antônio Carlos Jobim"
7"Apocalyptica"
8"Audioslave"
9"BackBeat"
10"Billy Cobham"
" + ], + "text/plain": [ + "shape: (10, 2)\n", + "┌──────────┬──────────────────────┐\n", + "│ column_0 ┆ column_1 │\n", + "│ --- ┆ --- │\n", + "│ i64 ┆ str │\n", + "╞══════════╪══════════════════════╡\n", + "│ 1 ┆ AC/DC │\n", + "│ 2 ┆ Accept │\n", + "│ 3 ┆ Aerosmith │\n", + "│ 4 ┆ Alanis Morissette │\n", + "│ 5 ┆ Alice In Chains │\n", + "│ 6 ┆ Antônio Carlos Jobim │\n", + "│ 7 ┆ Apocalyptica │\n", + "│ 8 ┆ Audioslave │\n", + "│ 9 ┆ BackBeat │\n", + "│ 10 ┆ Billy Cobham │\n", + "└──────────┴──────────────────────┘" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import polars as pl\n", "\n", @@ -138,10 +195,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "62791eab", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['ArtistId', 'Name']" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "[i[0] for i in cursor.description]" ] @@ -158,7 +226,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "0074d4d3", "metadata": {}, "outputs": [], @@ -181,7 +249,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "8020a73a", "metadata": {}, "outputs": [], @@ -202,10 +270,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "8a8354b2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[('US', 1.0, 3), ('UK', 0.6, 2), ('France', 0.8, 1)]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "con_new.execute(\"SELECT * FROM test\").fetchall()" ] @@ -220,7 +299,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "927e804d", "metadata": { "tags": [ @@ -278,10 +357,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "be55f957", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[('Bodies', 180035),\n", + " ('Vivo Isolado Do Mundo', 180035),\n", + " ('Elvis Ate America', 180166),\n", + " ('Remote Control', 180297),\n", + " ('Promises', 180401),\n", + " ('Emergency', 180427),\n", + " ('À Vontade (Live Mix)', 180636),\n", + " ('Hyperconectividade', 180636),\n", + " ('On Fire', 180636),\n", + " ('Fascinação', 180793)]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sql_query = \"SELECT name, milliseconds FROM track WHERE milliseconds > 1e3*3*60 ORDER BY milliseconds ASC LIMIT 10;\"\n", "cursor = con.execute(sql_query)\n", @@ -299,10 +398,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "3f894066", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[(340, 0.863),\n", + " (345, 1.11065),\n", + " (318, 1.6882166666666667),\n", + " (314, 1.69135),\n", + " (328, 1.8377666666666668)]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sql_groupby = \"SELECT albumid, AVG(milliseconds)/1e3/60 FROM track GROUP BY albumid ORDER BY AVG(milliseconds) ASC LIMIT 5;\"\n", "cursor = con.execute(sql_groupby)\n", @@ -324,10 +438,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "9824b70a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[(340, 0.863, \"Liszt - 12 Études D'Execution Transcendante\"),\n", + " (345, 1.11065, \"Monteverdi: L'Orfeo\"),\n", + " (318, 1.6882166666666667, 'SCRIABIN: Vers la flamme'),\n", + " (314, 1.69135, 'English Renaissance'),\n", + " (328, 1.8377666666666668, 'Charpentier: Divertissements, Airs & Concerts')]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sql_join = \"SELECT track.albumid, AVG(milliseconds)/1e3/60, album.title FROM track INNER JOIN album ON (track.albumid = album.albumid) GROUP BY album.albumid ORDER BY AVG(milliseconds) ASC LIMIT 5;\"\n", "cursor = con.execute(sql_join)\n", @@ -411,7 +540,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "f5c83909", "metadata": {}, "outputs": [], @@ -448,10 +577,45 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "fc89c429", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
+       "┃ TrackId ┃ Name                                    ┃ AlbumId ┃ MediaTypeId ┃ GenreId ┃ Composer                                                               ┃ Milliseconds ┃ Bytes    ┃ UnitPrice       ┃\n",
+       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
+       "│ !int64  │ !string                                 │ int64   │ !int64      │ int64   │ string                                                                 │ !int64       │ int64    │ !decimal(10, 2) │\n",
+       "├─────────┼─────────────────────────────────────────┼─────────┼─────────────┼─────────┼────────────────────────────────────────────────────────────────────────┼──────────────┼──────────┼─────────────────┤\n",
+       "│       1 │ For Those About To Rock (We Salute You) │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson                              │       343719 │ 11170334 │            0.99 │\n",
+       "│       2 │ Balls to the Wall                       │       2 │           2 │       1 │ NULL                                                                   │       342562 │  5510424 │            0.99 │\n",
+       "│       3 │ Fast As a Shark                         │       3 │           2 │       1 │ F. Baltes, S. Kaufman, U. Dirkscneider & W. Hoffman                    │       230619 │  3990994 │            0.99 │\n",
+       "│       4 │ Restless and Wild                       │       3 │           2 │       1 │ F. Baltes, R.A. Smith-Diesel, S. Kaufman, U. Dirkscneider & W. Hoffman │       252051 │  4331779 │            0.99 │\n",
+       "│       5 │ Princess of the Dawn                    │       3 │           2 │       1 │ Deaffy & R.A. Smith-Diesel                                             │       375418 │  6290521 │            0.99 │\n",
+       "└─────────┴─────────────────────────────────────────┴─────────┴─────────────┴─────────┴────────────────────────────────────────────────────────────────────────┴──────────────┴──────────┴─────────────────┘\n",
+       "
\n" + ], + "text/plain": [ + "┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n", + "┃\u001b[1m \u001b[0m\u001b[1mTrackId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mName\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mAlbumId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMediaTypeId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mGenreId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mComposer\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMilliseconds\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mBytes\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mUnitPrice\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\n", + "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n", + "│ \u001b[2m!int64\u001b[0m │ \u001b[2m!string\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2m!int64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2m!int64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2m!decimal(10, 2)\u001b[0m │\n", + "├─────────┼─────────────────────────────────────────┼─────────┼─────────────┼─────────┼────────────────────────────────────────────────────────────────────────┼──────────────┼──────────┼─────────────────┤\n", + "│ \u001b[1;36m1\u001b[0m │ \u001b[32mFor Those About To Rock (We Salute You)\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson \u001b[0m │ \u001b[1;36m343719\u001b[0m │ \u001b[1;36m11170334\u001b[0m │ \u001b[1;36m0.99\u001b[0m │\n", + "│ \u001b[1;36m2\u001b[0m │ \u001b[32mBalls to the Wall \u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[1;36m342562\u001b[0m │ \u001b[1;36m5510424\u001b[0m │ \u001b[1;36m0.99\u001b[0m │\n", + "│ \u001b[1;36m3\u001b[0m │ \u001b[32mFast As a Shark \u001b[0m │ \u001b[1;36m3\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mF. Baltes, S. Kaufman, U. Dirkscneider & W. Hoffman \u001b[0m │ \u001b[1;36m230619\u001b[0m │ \u001b[1;36m3990994\u001b[0m │ \u001b[1;36m0.99\u001b[0m │\n", + "│ \u001b[1;36m4\u001b[0m │ \u001b[32mRestless and Wild \u001b[0m │ \u001b[1;36m3\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mF. Baltes, R.A. Smith-Diesel, S. Kaufman, U. Dirkscneider & W. Hoffman\u001b[0m │ \u001b[1;36m252051\u001b[0m │ \u001b[1;36m4331779\u001b[0m │ \u001b[1;36m0.99\u001b[0m │\n", + "│ \u001b[1;36m5\u001b[0m │ \u001b[32mPrincess of the Dawn \u001b[0m │ \u001b[1;36m3\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mDeaffy & R.A. Smith-Diesel \u001b[0m │ \u001b[1;36m375418\u001b[0m │ \u001b[1;36m6290521\u001b[0m │ \u001b[1;36m0.99\u001b[0m │\n", + "└─────────┴─────────────────────────────────────────┴─────────┴─────────────┴─────────┴────────────────────────────────────────────────────────────────────────┴──────────────┴──────────┴─────────────────┘" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import ibis\n", "\n", @@ -471,10 +635,45 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "5e5a482b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
+       "┃ TrackId ┃ Name                                                     ┃ AlbumId ┃ MediaTypeId ┃ GenreId ┃ Composer           ┃ Milliseconds ┃ Bytes   ┃ UnitPrice       ┃ mean_mins_track ┃\n",
+       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
+       "│ !int64  │ !string                                                  │ int64   │ !int64      │ int64   │ string             │ !int64       │ int64   │ !decimal(10, 2) │ float64         │\n",
+       "├─────────┼──────────────────────────────────────────────────────────┼─────────┼─────────────┼─────────┼────────────────────┼──────────────┼─────────┼─────────────────┼─────────────────┤\n",
+       "│    3496 │ Étude 1, In C Major - Preludio (Presto) - Liszt          │     340 │           4 │      24 │ NULL               │        51780 │ 2229617 │            0.99 │        0.863000 │\n",
+       "│    3501 │ L'orfeo, Act 3, Sinfonia (Orchestra)                     │     345 │           2 │      24 │ Claudio Monteverdi │        66639 │ 1189062 │            0.99 │        1.110650 │\n",
+       "│    3452 │ SCRIABIN: Prelude in B Major, Op. 11, No. 11             │     318 │           4 │      24 │ NULL               │       101293 │ 3819535 │            0.99 │        1.688217 │\n",
+       "│    3448 │ Lamentations of Jeremiah, First Set \\ Incipit Lamentatio │     314 │           2 │      24 │ Thomas Tallis      │        69194 │ 1208080 │            0.99 │        1.691350 │\n",
+       "│    3492 │ Sing Joyfully                                            │     314 │           2 │      24 │ William Byrd       │       133768 │ 2256484 │            0.99 │        1.691350 │\n",
+       "└─────────┴──────────────────────────────────────────────────────────┴─────────┴─────────────┴─────────┴────────────────────┴──────────────┴─────────┴─────────────────┴─────────────────┘\n",
+       "
\n" + ], + "text/plain": [ + "┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n", + "┃\u001b[1m \u001b[0m\u001b[1mTrackId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mName\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mAlbumId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMediaTypeId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mGenreId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mComposer\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMilliseconds\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mBytes\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mUnitPrice\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mmean_mins_track\u001b[0m\u001b[1m \u001b[0m┃\n", + "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n", + "│ \u001b[2m!int64\u001b[0m │ \u001b[2m!string\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2m!int64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2m!int64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2m!decimal(10, 2)\u001b[0m │ \u001b[2mfloat64\u001b[0m │\n", + "├─────────┼──────────────────────────────────────────────────────────┼─────────┼─────────────┼─────────┼────────────────────┼──────────────┼─────────┼─────────────────┼─────────────────┤\n", + "│ \u001b[1;36m3496\u001b[0m │ \u001b[32mÉtude 1, In C Major - Preludio (Presto) - Liszt \u001b[0m │ \u001b[1;36m340\u001b[0m │ \u001b[1;36m4\u001b[0m │ \u001b[1;36m24\u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[1;36m51780\u001b[0m │ \u001b[1;36m2229617\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m0.863000\u001b[0m │\n", + "│ \u001b[1;36m3501\u001b[0m │ \u001b[32mL'orfeo, Act 3, Sinfonia (Orchestra) \u001b[0m │ \u001b[1;36m345\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m24\u001b[0m │ \u001b[32mClaudio Monteverdi\u001b[0m │ \u001b[1;36m66639\u001b[0m │ \u001b[1;36m1189062\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m1.110650\u001b[0m │\n", + "│ \u001b[1;36m3452\u001b[0m │ \u001b[32mSCRIABIN: Prelude in B Major, Op. 11, No. 11 \u001b[0m │ \u001b[1;36m318\u001b[0m │ \u001b[1;36m4\u001b[0m │ \u001b[1;36m24\u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[1;36m101293\u001b[0m │ \u001b[1;36m3819535\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m1.688217\u001b[0m │\n", + "│ \u001b[1;36m3448\u001b[0m │ \u001b[32mLamentations of Jeremiah, First Set \\ Incipit Lamentatio\u001b[0m │ \u001b[1;36m314\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m24\u001b[0m │ \u001b[32mThomas Tallis \u001b[0m │ \u001b[1;36m69194\u001b[0m │ \u001b[1;36m1208080\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m1.691350\u001b[0m │\n", + "│ \u001b[1;36m3492\u001b[0m │ \u001b[32mSing Joyfully \u001b[0m │ \u001b[1;36m314\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m24\u001b[0m │ \u001b[32mWilliam Byrd \u001b[0m │ \u001b[1;36m133768\u001b[0m │ \u001b[1;36m2256484\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m1.691350\u001b[0m │\n", + "└─────────┴──────────────────────────────────────────────────────────┴─────────┴─────────────┴─────────┴────────────────────┴──────────────┴─────────┴─────────────────┴─────────────────┘" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "track.group_by(\"AlbumId\").mutate(\n", " mean_mins_track=track.Milliseconds.mean() / 1e3 / 60\n", @@ -483,10 +682,57 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "39c80365", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
+       "┃ TrackId ┃ Name                                    ┃ AlbumId ┃ MediaTypeId ┃ GenreId ┃ Composer                                  ┃ Milliseconds ┃ Bytes    ┃ UnitPrice       ┃ mean_mins_track ┃\n",
+       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
+       "│ !int64  │ !string                                 │ int64   │ !int64      │ int64   │ string                                    │ !int64       │ int64    │ !decimal(10, 2) │ float64         │\n",
+       "├─────────┼─────────────────────────────────────────┼─────────┼─────────────┼─────────┼───────────────────────────────────────────┼──────────────┼──────────┼─────────────────┼─────────────────┤\n",
+       "│       1 │ For Those About To Rock (We Salute You) │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       343719 │ 11170334 │            0.99 │        4.000692 │\n",
+       "│       6 │ Put The Finger On You                   │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       205662 │  6713451 │            0.99 │        4.000692 │\n",
+       "│       7 │ Let's Get It Up                         │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       233926 │  7636561 │            0.99 │        4.000692 │\n",
+       "│       8 │ Inject The Venom                        │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       210834 │  6852860 │            0.99 │        4.000692 │\n",
+       "│       9 │ Snowballed                              │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       203102 │  6599424 │            0.99 │        4.000692 │\n",
+       "│      10 │ Evil Walks                              │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       263497 │  8611245 │            0.99 │        4.000692 │\n",
+       "│      11 │ C.O.D.                                  │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       199836 │  6566314 │            0.99 │        4.000692 │\n",
+       "│      12 │ Breaking The Rules                      │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       263288 │  8596840 │            0.99 │        4.000692 │\n",
+       "│      13 │ Night Of The Long Knives                │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       205688 │  6706347 │            0.99 │        4.000692 │\n",
+       "│      14 │ Spellbound                              │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       270863 │  8817038 │            0.99 │        4.000692 │\n",
+       "│       … │ …                                       │       … │           … │       … │ …                                         │            … │        … │               … │               … │\n",
+       "└─────────┴─────────────────────────────────────────┴─────────┴─────────────┴─────────┴───────────────────────────────────────────┴──────────────┴──────────┴─────────────────┴─────────────────┘\n",
+       "
\n" + ], + "text/plain": [ + "┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n", + "┃\u001b[1m \u001b[0m\u001b[1mTrackId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mName\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mAlbumId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMediaTypeId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mGenreId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mComposer\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMilliseconds\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mBytes\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mUnitPrice\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mmean_mins_track\u001b[0m\u001b[1m \u001b[0m┃\n", + "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n", + "│ \u001b[2m!int64\u001b[0m │ \u001b[2m!string\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2m!int64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2m!int64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2m!decimal(10, 2)\u001b[0m │ \u001b[2mfloat64\u001b[0m │\n", + "├─────────┼─────────────────────────────────────────┼─────────┼─────────────┼─────────┼───────────────────────────────────────────┼──────────────┼──────────┼─────────────────┼─────────────────┤\n", + "│ \u001b[1;36m1\u001b[0m │ \u001b[32mFor Those About To Rock (We Salute You)\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m343719\u001b[0m │ \u001b[1;36m11170334\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", + "│ \u001b[1;36m6\u001b[0m │ \u001b[32mPut The Finger On You \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m205662\u001b[0m │ \u001b[1;36m6713451\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", + "│ \u001b[1;36m7\u001b[0m │ \u001b[32mLet's Get It Up \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m233926\u001b[0m │ \u001b[1;36m7636561\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", + "│ \u001b[1;36m8\u001b[0m │ \u001b[32mInject The Venom \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m210834\u001b[0m │ \u001b[1;36m6852860\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", + "│ \u001b[1;36m9\u001b[0m │ \u001b[32mSnowballed \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m203102\u001b[0m │ \u001b[1;36m6599424\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", + "│ \u001b[1;36m10\u001b[0m │ \u001b[32mEvil Walks \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m263497\u001b[0m │ \u001b[1;36m8611245\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", + "│ \u001b[1;36m11\u001b[0m │ \u001b[32mC.O.D. \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m199836\u001b[0m │ \u001b[1;36m6566314\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", + "│ \u001b[1;36m12\u001b[0m │ \u001b[32mBreaking The Rules \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m263288\u001b[0m │ \u001b[1;36m8596840\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", + "│ \u001b[1;36m13\u001b[0m │ \u001b[32mNight Of The Long Knives \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m205688\u001b[0m │ \u001b[1;36m6706347\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", + "│ \u001b[1;36m14\u001b[0m │ \u001b[32mSpellbound \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m270863\u001b[0m │ \u001b[1;36m8817038\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", + "│ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │\n", + "└─────────┴─────────────────────────────────────────┴─────────┴─────────────┴─────────┴───────────────────────────────────────────┴──────────────┴──────────┴─────────────────┴─────────────────┘" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "track.group_by(\"AlbumId\").mutate(mean_mins_track=track.Milliseconds.mean() / 1e3 / 60)" ] @@ -501,10 +747,57 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "02e6602a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓\n",
+       "┃ TrackId ┃ Name                                    ┃ AlbumId ┃ MediaTypeId ┃ GenreId ┃ Composer                                                               ┃ Milliseconds ┃ Bytes    ┃ UnitPrice      ┃ Name_right ┃\n",
+       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩\n",
+       "│ int64   │ string                                  │ int64   │ int64       │ int64   │ string                                                                 │ int64        │ int64    │ decimal(10, 2) │ string     │\n",
+       "├─────────┼─────────────────────────────────────────┼─────────┼─────────────┼─────────┼────────────────────────────────────────────────────────────────────────┼──────────────┼──────────┼────────────────┼────────────┤\n",
+       "│       1 │ For Those About To Rock (We Salute You) │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson                              │       343719 │ 11170334 │           0.99 │ Rock       │\n",
+       "│       2 │ Balls to the Wall                       │       2 │           2 │       1 │ NULL                                                                   │       342562 │  5510424 │           0.99 │ Rock       │\n",
+       "│       3 │ Fast As a Shark                         │       3 │           2 │       1 │ F. Baltes, S. Kaufman, U. Dirkscneider & W. Hoffman                    │       230619 │  3990994 │           0.99 │ Rock       │\n",
+       "│       4 │ Restless and Wild                       │       3 │           2 │       1 │ F. Baltes, R.A. Smith-Diesel, S. Kaufman, U. Dirkscneider & W. Hoffman │       252051 │  4331779 │           0.99 │ Rock       │\n",
+       "│       5 │ Princess of the Dawn                    │       3 │           2 │       1 │ Deaffy & R.A. Smith-Diesel                                             │       375418 │  6290521 │           0.99 │ Rock       │\n",
+       "│       6 │ Put The Finger On You                   │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson                              │       205662 │  6713451 │           0.99 │ Rock       │\n",
+       "│       7 │ Let's Get It Up                         │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson                              │       233926 │  7636561 │           0.99 │ Rock       │\n",
+       "│       8 │ Inject The Venom                        │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson                              │       210834 │  6852860 │           0.99 │ Rock       │\n",
+       "│       9 │ Snowballed                              │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson                              │       203102 │  6599424 │           0.99 │ Rock       │\n",
+       "│      10 │ Evil Walks                              │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson                              │       263497 │  8611245 │           0.99 │ Rock       │\n",
+       "│       … │ …                                       │       … │           … │       … │ …                                                                      │            … │        … │              … │ …          │\n",
+       "└─────────┴─────────────────────────────────────────┴─────────┴─────────────┴─────────┴────────────────────────────────────────────────────────────────────────┴──────────────┴──────────┴────────────────┴────────────┘\n",
+       "
\n" + ], + "text/plain": [ + "┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓\n", + "┃\u001b[1m \u001b[0m\u001b[1mTrackId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mName\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mAlbumId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMediaTypeId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mGenreId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mComposer\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMilliseconds\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mBytes\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mUnitPrice\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mName_right\u001b[0m\u001b[1m \u001b[0m┃\n", + "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩\n", + "│ \u001b[2mint64\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mdecimal(10, 2)\u001b[0m │ \u001b[2mstring\u001b[0m │\n", + "├─────────┼─────────────────────────────────────────┼─────────┼─────────────┼─────────┼────────────────────────────────────────────────────────────────────────┼──────────────┼──────────┼────────────────┼────────────┤\n", + "│ \u001b[1;36m1\u001b[0m │ \u001b[32mFor Those About To Rock (We Salute You)\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson \u001b[0m │ \u001b[1;36m343719\u001b[0m │ \u001b[1;36m11170334\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", + "│ \u001b[1;36m2\u001b[0m │ \u001b[32mBalls to the Wall \u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[1;36m342562\u001b[0m │ \u001b[1;36m5510424\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", + "│ \u001b[1;36m3\u001b[0m │ \u001b[32mFast As a Shark \u001b[0m │ \u001b[1;36m3\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mF. Baltes, S. Kaufman, U. Dirkscneider & W. Hoffman \u001b[0m │ \u001b[1;36m230619\u001b[0m │ \u001b[1;36m3990994\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", + "│ \u001b[1;36m4\u001b[0m │ \u001b[32mRestless and Wild \u001b[0m │ \u001b[1;36m3\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mF. Baltes, R.A. Smith-Diesel, S. Kaufman, U. Dirkscneider & W. Hoffman\u001b[0m │ \u001b[1;36m252051\u001b[0m │ \u001b[1;36m4331779\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", + "│ \u001b[1;36m5\u001b[0m │ \u001b[32mPrincess of the Dawn \u001b[0m │ \u001b[1;36m3\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mDeaffy & R.A. Smith-Diesel \u001b[0m │ \u001b[1;36m375418\u001b[0m │ \u001b[1;36m6290521\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", + "│ \u001b[1;36m6\u001b[0m │ \u001b[32mPut The Finger On You \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson \u001b[0m │ \u001b[1;36m205662\u001b[0m │ \u001b[1;36m6713451\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", + "│ \u001b[1;36m7\u001b[0m │ \u001b[32mLet's Get It Up \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson \u001b[0m │ \u001b[1;36m233926\u001b[0m │ \u001b[1;36m7636561\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", + "│ \u001b[1;36m8\u001b[0m │ \u001b[32mInject The Venom \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson \u001b[0m │ \u001b[1;36m210834\u001b[0m │ \u001b[1;36m6852860\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", + "│ \u001b[1;36m9\u001b[0m │ \u001b[32mSnowballed \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson \u001b[0m │ \u001b[1;36m203102\u001b[0m │ \u001b[1;36m6599424\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", + "│ \u001b[1;36m10\u001b[0m │ \u001b[32mEvil Walks \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson \u001b[0m │ \u001b[1;36m263497\u001b[0m │ \u001b[1;36m8611245\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", + "│ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │\n", + "└─────────┴─────────────────────────────────────────┴─────────┴─────────────┴─────────┴────────────────────────────────────────────────────────────────────────┴──────────────┴──────────┴────────────────┴────────────┘" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "genre = connection.table(\"genre\")\n", "genre_and_track = track.inner_join(\n", @@ -557,7 +850,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "325c8789", "metadata": { "tags": [ @@ -577,7 +870,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "9117af26", "metadata": {}, "outputs": [], @@ -604,7 +897,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "e8277743", "metadata": {}, "outputs": [], @@ -625,7 +918,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "2097c995", "metadata": {}, "outputs": [], @@ -652,10 +945,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "14c14fc8", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[(1, 'Deadpond', 'Dive Wilson', None),\n", + " (2, 'Spider-Boy', 'Pedro Parqueador', None),\n", + " (3, 'Rusty-Man', 'Tommy Sharp', 48),\n", + " (4, 'Ms Amazing', 'Barjabeen Bhabra', 17)]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "con = sqlite3.connect(Path(\"data/hero.db\"))\n", "\n", @@ -674,10 +981,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "afa69365", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "name='Deadpond' secret_name='Dive Wilson' id=1 age=None\n", + "name='Spider-Boy' secret_name='Pedro Parqueador' id=2 age=None\n", + "name='Rusty-Man' secret_name='Tommy Sharp' id=3 age=48\n", + "name='Ms Amazing' secret_name='Barjabeen Bhabra' id=4 age=17\n" + ] + } + ], "source": [ "from sqlmodel import select\n", "\n", @@ -698,10 +1016,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "8fae7081", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "name='Deadpond' secret_name='Dive Wilson' id=1 age=None\n", + "name='Spider-Boy' secret_name='Pedro Parqueador' id=2 age=None\n", + "name='Rusty-Man' secret_name='Tommy Sharp' id=3 age=48\n", + "name='Ms Amazing' secret_name='Barjabeen Bhabra' id=4 age=17\n" + ] + } + ], "source": [ "with Session(engine) as session:\n", " statement = select(Hero)\n", @@ -721,10 +1050,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "b2dbde03", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "name='Ms Amazing' secret_name='Barjabeen Bhabra' id=4 age=17\n", + "name='Rusty-Man' secret_name='Tommy Sharp' id=3 age=48\n" + ] + } + ], "source": [ "with Session(engine) as session:\n", " statement = select(Hero).where(Hero.age < 100).limit(2).order_by(Hero.age)\n", @@ -744,7 +1082,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "id": "349e5d3d", "metadata": { "tags": [ @@ -797,7 +1135,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/dates-and-times.ipynb b/dates-and-times.ipynb index 4fe0c68..1ef38ab 100644 --- a/dates-and-times.ipynb +++ b/dates-and-times.ipynb @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "51a55374", "metadata": { "tags": [ @@ -74,10 +74,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "84829a6b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2026-04-28 13:36:00.720344\n" + ] + } + ], "source": [ "from datetime import datetime\n", "\n", @@ -121,10 +129,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "fc224a47", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2019-11-28 00:00:00\n" + ] + } + ], "source": [ "specific_datetime = datetime(2019, 11, 28)\n", "print(specific_datetime)" @@ -150,10 +166,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "4558d476", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "datetime.datetime(2002, 2, 16, 0, 0)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "date_string = \"16 February in 2002\"\n", "datetime.strptime(date_string, \"%d %B in %Y\")" @@ -169,10 +196,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "fb90ac84", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "datetime.datetime(2002, 2, 16, 0, 0)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "date_string = \"16 Feb in 2002\"\n", "datetime.strptime(date_string, \"%d %b in %Y\")" @@ -188,10 +226,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "7a568f5b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2002-02-03 00:00:00\n", + "2002-02-03 00:00:00\n" + ] + } + ], "source": [ "from dateutil.parser import parse\n", "\n", @@ -211,10 +258,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "d6b5e3c3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'Tuesday, 04, 2026'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "now.strftime(\"%A, %m, %Y\")" ] @@ -273,10 +331,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "39ba17e6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "now > specific_datetime" ] @@ -291,10 +360,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "fda57a44", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2309 days, 13:36:00.720344\n" + ] + } + ], "source": [ "time_diff = now - datetime(year=2020, month=1, day=1)\n", "print(time_diff)" @@ -310,10 +387,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "e1e25736", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "datetime.timedelta" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "type(time_diff)" ] @@ -361,10 +449,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "ed526fbc", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "array('2020-01-01', dtype='datetime64[D]')" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import numpy as np\n", "\n", @@ -382,10 +481,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "537895c5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "array(['2020-01-01', '2020-01-02', '2020-01-03', '2020-01-04',\n", + " '2020-01-05', '2020-01-06', '2020-01-07', '2020-01-08',\n", + " '2020-01-09', '2020-01-10', '2020-01-11', '2020-01-12',\n", + " '2020-01-13', '2020-01-14', '2020-01-15', '2020-01-16',\n", + " '2020-01-17', '2020-01-18', '2020-01-19', '2020-01-20',\n", + " '2020-01-21', '2020-01-22', '2020-01-23', '2020-01-24',\n", + " '2020-01-25', '2020-01-26', '2020-01-27', '2020-01-28',\n", + " '2020-01-29', '2020-01-30', '2020-01-31', '2020-02-01'],\n", + " dtype='datetime64[D]')" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "date + range(32)" ] @@ -402,10 +520,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "cd7a15e3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "np.datetime64('2020-01-01T09:00')" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "np.datetime64(\"2020-01-01 09:00\")" ] @@ -436,10 +565,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "48f0e9c6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Timestamp('2020-02-16 00:00:00')" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import pandas as pd\n", "\n", @@ -469,10 +609,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "462b26da", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "DatetimeIndex(['2020-02-16', '2020-02-17', '2020-02-18', '2020-02-19',\n", + " '2020-02-20', '2020-02-21', '2020-02-22', '2020-02-23',\n", + " '2020-02-24', '2020-02-25', '2020-02-26', '2020-02-27'],\n", + " dtype='datetime64[ns]', freq=None)" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "date + pd.to_timedelta(np.arange(12), \"D\")" ] @@ -489,10 +643,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "10e71325", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "DatetimeIndex(['2018-01-01', '2018-01-02', '2018-01-03', '2018-01-04',\n", + " '2018-01-05', '2018-01-06', '2018-01-07', '2018-01-08'],\n", + " dtype='datetime64[ns]', freq='D')" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pd.date_range(start=\"2018/1/1\", end=\"2018/1/8\")" ] @@ -507,10 +674,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "291ace2c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "DatetimeIndex(['2018-01-01 00:00:00', '2018-01-01 01:00:00',\n", + " '2018-01-01 02:00:00'],\n", + " dtype='datetime64[ns]', freq='h')" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pd.date_range(\"2018-01-01\", periods=3, freq=\"h\")" ] @@ -525,10 +705,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "6703682c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "DatetimeIndex(['2017-12-31 16:00:00-08:00', '2017-12-31 17:00:00-08:00',\n", + " '2017-12-31 18:00:00-08:00'],\n", + " dtype='datetime64[ns, US/Pacific]', freq='h')" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dti = pd.date_range(\"2018-01-01\", periods=3, freq=\"h\").tz_localize(\"UTC\")\n", "dti.tz_convert(\"US/Pacific\")" @@ -544,10 +737,79 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "dd00df7f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dateVacancies (ICT), thousands
02001 MAY568
12001 JUN563
22001 JUL554
32001 AUG554
42001 SEP536
\n", + "
" + ], + "text/plain": [ + " date Vacancies (ICT), thousands\n", + "0 2001 MAY 568\n", + "1 2001 JUN 563\n", + "2 2001 JUL 554\n", + "3 2001 AUG 554\n", + "4 2001 SEP 536" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import requests\n", "\n", @@ -572,10 +834,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "247d9725", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 281 entries, 0 to 280\n", + "Data columns (total 2 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 date 281 non-null object\n", + " 1 Vacancies (ICT), thousands 281 non-null int64 \n", + "dtypes: int64(1), object(1)\n", + "memory usage: 4.5+ KB\n" + ] + } + ], "source": [ "df.info()" ] @@ -590,10 +868,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "b90f8038", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60965/3535541307.py:1: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.\n", + " df[\"date\"] = pd.to_datetime(df[\"date\"])\n" + ] + }, + { + "data": { + "text/plain": [ + "0 2001-05-01\n", + "1 2001-06-01\n", + "2 2001-07-01\n", + "3 2001-08-01\n", + "4 2001-09-01\n", + "Name: date, dtype: datetime64[ns]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[\"date\"] = pd.to_datetime(df[\"date\"])\n", "df[\"date\"].head()" @@ -611,10 +913,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "05d056ae", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 1, '19, 22\n", + "1 1, '19, 23\n", + "Name: date, dtype: object" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "small_df = pd.DataFrame({\"date\": [\"1, '19, 22\", \"1, '19, 23\"], \"values\": [\"1\", \"2\"]})\n", "small_df[\"date\"]" @@ -630,10 +945,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "514c9052", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 2019-01-22\n", + "1 2019-01-23\n", + "Name: date, dtype: datetime64[ns]" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pd.to_datetime(small_df[\"date\"], format=\"%m, '%y, %d\")" ] @@ -650,10 +978,79 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "ac3addbc", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dateVacancies (ICT), thousands
02001-05-31568
12001-06-30563
22001-07-31554
32001-08-31554
42001-09-30536
\n", + "
" + ], + "text/plain": [ + " date Vacancies (ICT), thousands\n", + "0 2001-05-31 568\n", + "1 2001-06-30 563\n", + "2 2001-07-31 554\n", + "3 2001-08-31 554\n", + "4 2001-09-30 536" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[\"date\"] = df[\"date\"] + pd.offsets.MonthEnd()\n", "df.head()" @@ -679,10 +1076,38 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "id": "a6c3d2d9", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using `dt.day_name()`\n", + "0 Thursday\n", + "1 Saturday\n", + "2 Tuesday\n", + "3 Friday\n", + "4 Sunday\n", + "Name: date, dtype: object\n", + "Using `dt.isocalendar()`\n", + " year week day\n", + "0 2001 22 4\n", + "1 2001 26 6\n", + "2 2001 31 2\n", + "3 2001 35 5\n", + "4 2001 39 7\n", + "Using `dt.month`\n", + "0 5\n", + "1 6\n", + "2 7\n", + "3 8\n", + "4 9\n", + "Name: date, dtype: int32\n" + ] + } + ], "source": [ "print(\"Using `dt.day_name()`\")\n", "print(df[\"date\"].dt.day_name().head())\n", @@ -704,10 +1129,78 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "id": "e0a4f68d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Vacancies (ICT), thousands
date
2001-05-31568
2001-06-30563
2001-07-31554
2001-08-31554
2001-09-30536
\n", + "
" + ], + "text/plain": [ + " Vacancies (ICT), thousands\n", + "date \n", + "2001-05-31 568\n", + "2001-06-30 563\n", + "2001-07-31 554\n", + "2001-08-31 554\n", + "2001-09-30 536" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = df.set_index(\"date\")\n", "df.head()" @@ -723,10 +1216,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "acf1ae60", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "DatetimeIndex(['2001-05-31', '2001-06-30', '2001-07-31', '2001-08-31',\n", + " '2001-09-30'],\n", + " dtype='datetime64[ns]', name='date', freq=None)" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.index[:5]" ] @@ -741,10 +1247,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "9146c99d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60965/2067773505.py:1: FutureWarning: 'M' is deprecated and will be removed in a future version, please use 'ME' instead.\n", + " df = df.asfreq(\"M\")\n" + ] + }, + { + "data": { + "text/plain": [ + "DatetimeIndex(['2001-05-31', '2001-06-30', '2001-07-31', '2001-08-31',\n", + " '2001-09-30'],\n", + " dtype='datetime64[ns]', name='date', freq='ME')" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = df.asfreq(\"M\")\n", "df.index[:5]" @@ -794,10 +1321,1352 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "id": "b4c5f841", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:36:02.756256\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "df.plot();" ] @@ -818,10 +2687,181 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "id": "e56ba5c4", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60965/311401334.py:1: FutureWarning: 'A' is deprecated and will be removed in a future version, please use 'YE' instead.\n", + " df.resample(\"A\").mean()\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Vacancies (ICT), thousands
date
2001-12-31540.625000
2002-12-31517.500000
2003-12-31504.166667
2004-12-31551.916667
2005-12-31544.666667
2006-12-31529.500000
2007-12-31576.333333
2008-12-31544.583333
2009-12-31402.750000
2010-12-31424.166667
2011-12-31413.250000
2012-12-31423.916667
2013-12-31480.250000
2014-12-31592.416667
2015-12-31655.166667
2016-12-31671.250000
2017-12-31704.750000
2018-12-31742.666667
2019-12-31734.166667
2020-12-31487.500000
2021-12-31843.416667
2022-12-311092.083333
2023-12-31894.500000
2024-12-31767.888889
\n", + "
" + ], + "text/plain": [ + " Vacancies (ICT), thousands\n", + "date \n", + "2001-12-31 540.625000\n", + "2002-12-31 517.500000\n", + "2003-12-31 504.166667\n", + "2004-12-31 551.916667\n", + "2005-12-31 544.666667\n", + "2006-12-31 529.500000\n", + "2007-12-31 576.333333\n", + "2008-12-31 544.583333\n", + "2009-12-31 402.750000\n", + "2010-12-31 424.166667\n", + "2011-12-31 413.250000\n", + "2012-12-31 423.916667\n", + "2013-12-31 480.250000\n", + "2014-12-31 592.416667\n", + "2015-12-31 655.166667\n", + "2016-12-31 671.250000\n", + "2017-12-31 704.750000\n", + "2018-12-31 742.666667\n", + "2019-12-31 734.166667\n", + "2020-12-31 487.500000\n", + "2021-12-31 843.416667\n", + "2022-12-31 1092.083333\n", + "2023-12-31 894.500000\n", + "2024-12-31 767.888889" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.resample(\"A\").mean()" ] @@ -836,10 +2876,94 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "id": "fbbbcdff", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Vacancies (ICT), thousands
meanstd
date
2001-12-31540.62500022.398581
2006-12-31529.55000020.434621
2011-12-31472.21666777.919796
2016-12-31564.60000099.829210
2021-12-31702.500000164.019480
\n", + "
" + ], + "text/plain": [ + " Vacancies (ICT), thousands \n", + " mean std\n", + "date \n", + "2001-12-31 540.625000 22.398581\n", + "2006-12-31 529.550000 20.434621\n", + "2011-12-31 472.216667 77.919796\n", + "2016-12-31 564.600000 99.829210\n", + "2021-12-31 702.500000 164.019480" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.resample(\"5YE\").agg([\"mean\", \"std\"]).head()" ] @@ -854,10 +2978,111 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "id": "9a48a45f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Vacancies (ICT), thousands
date
2001-05-31568.0
2001-06-01NaN
2001-06-02NaN
2001-06-03NaN
2001-06-04NaN
......
2024-09-26NaN
2024-09-27NaN
2024-09-28NaN
2024-09-29NaN
2024-09-30727.0
\n", + "

8524 rows × 1 columns

\n", + "
" + ], + "text/plain": [ + " Vacancies (ICT), thousands\n", + "date \n", + "2001-05-31 568.0\n", + "2001-06-01 NaN\n", + "2001-06-02 NaN\n", + "2001-06-03 NaN\n", + "2001-06-04 NaN\n", + "... ...\n", + "2024-09-26 NaN\n", + "2024-09-27 NaN\n", + "2024-09-28 NaN\n", + "2024-09-29 NaN\n", + "2024-09-30 727.0\n", + "\n", + "[8524 rows x 1 columns]" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.resample(\"D\").asfreq()" ] @@ -872,10 +3097,83 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "id": "d3ac1789", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Vacancies (ICT), thousands
date
2001-05-31568.000000
2001-06-01567.833333
2001-06-02567.666667
2001-06-03567.500000
2001-06-04NaN
2001-06-05NaN
\n", + "
" + ], + "text/plain": [ + " Vacancies (ICT), thousands\n", + "date \n", + "2001-05-31 568.000000\n", + "2001-06-01 567.833333\n", + "2001-06-02 567.666667\n", + "2001-06-03 567.500000\n", + "2001-06-04 NaN\n", + "2001-06-05 NaN" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.resample(\"D\").interpolate(method=\"linear\", limit_direction=\"forward\", limit=3)[:6]" ] @@ -890,10 +3188,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "id": "51647c56", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "[*********************100%***********************] 1 of 1 completed" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], "source": [ "# Get stock market data\n", "import yfinance as yf\n", @@ -905,20 +3219,1640 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "id": "ab1efae3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
'color'
'#bc80bd'
'#fb8072'
'#b3de69'
'#fdb462'
'#fccde5'
'#8dd3c7'
'#ffed6f'
'#bebada'
'#80b1d3'
'#ccebc5'
'#d9d9d9'
" + ], + "text/plain": [ + "cycler('color', ['#bc80bd', '#fb8072', '#b3de69', '#fdb462', '#fccde5', '#8dd3c7', '#ffed6f', '#bebada', '#80b1d3', '#ccebc5', '#d9d9d9'])" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "plt.rcParams[\"axes.prop_cycle\"]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "id": "fa0c9973", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:36:08.720742\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "from itertools import cycle\n", "\n", @@ -957,10 +4891,111 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "id": "1ddc4fb2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Vacancies (ICT), thousands
date
2001-05-31NaN
2001-06-30565.5
2001-07-31558.5
2001-08-31554.0
2001-09-30545.0
......
2024-05-31776.5
2024-06-30760.0
2024-07-31748.0
2024-08-31737.0
2024-09-30729.5
\n", + "

281 rows × 1 columns

\n", + "
" + ], + "text/plain": [ + " Vacancies (ICT), thousands\n", + "date \n", + "2001-05-31 NaN\n", + "2001-06-30 565.5\n", + "2001-07-31 558.5\n", + "2001-08-31 554.0\n", + "2001-09-30 545.0\n", + "... ...\n", + "2024-05-31 776.5\n", + "2024-06-30 760.0\n", + "2024-07-31 748.0\n", + "2024-08-31 737.0\n", + "2024-09-30 729.5\n", + "\n", + "[281 rows x 1 columns]" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.rolling(2).mean()" ] @@ -983,10 +5018,111 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "id": "0ea9c8ce", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Vacancies (ICT), thousands
date
2001-05-31568.000000
2001-06-30565.222222
2001-07-31560.622951
2001-08-31558.379404
2001-09-30551.722037
......
2024-05-31813.183347
2024-06-30801.346677
2024-07-31789.477342
2024-08-31777.981873
2024-09-30767.785499
\n", + "

281 rows × 1 columns

\n", + "
" + ], + "text/plain": [ + " Vacancies (ICT), thousands\n", + "date \n", + "2001-05-31 568.000000\n", + "2001-06-30 565.222222\n", + "2001-07-31 560.622951\n", + "2001-08-31 558.379404\n", + "2001-09-30 551.722037\n", + "... ...\n", + "2024-05-31 813.183347\n", + "2024-06-30 801.346677\n", + "2024-07-31 789.477342\n", + "2024-08-31 777.981873\n", + "2024-09-30 767.785499\n", + "\n", + "[281 rows x 1 columns]" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.ewm(alpha=0.2).mean()" ] @@ -1001,10 +5137,3078 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "id": "0af7b5e6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:36:08.899458\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "fig, ax = plt.subplots()\n", "roll_num = 28\n", @@ -1031,10 +8235,2366 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "id": "134199ae", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:36:08.991604\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "roll = xf[\"Close\"].rolling(50, center=True)\n", "\n", @@ -1057,10 +10617,92 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "id": "3078fbb4", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Vacancies (ICT), thousandslead (12 months)lag (3 months)
date
2001-05-31568518.0NaN
2001-06-30563514.0NaN
2001-07-31554517.0NaN
2001-08-31554517.0568.0
2001-09-30536519.0563.0
\n", + "
" + ], + "text/plain": [ + " Vacancies (ICT), thousands lead (12 months) lag (3 months)\n", + "date \n", + "2001-05-31 568 518.0 NaN\n", + "2001-06-30 563 514.0 NaN\n", + "2001-07-31 554 517.0 NaN\n", + "2001-08-31 554 517.0 568.0\n", + "2001-09-30 536 519.0 563.0" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "lead = 12\n", "lag = 3\n", @@ -1072,10 +10714,1831 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "id": "18b9afb3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:36:09.071401\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "df.iloc[100:300, :].plot();" ] @@ -1103,7 +12566,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/exploratory-data-analysis.ipynb b/exploratory-data-analysis.ipynb index 2bf323b..b08137e 100644 --- a/exploratory-data-analysis.ipynb +++ b/exploratory-data-analysis.ipynb @@ -28,10 +28,51 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "a3377aa6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "import pandas as pd\n", "from lets_plot import *\n", @@ -44,7 +85,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "51a55374", "metadata": { "tags": [ @@ -102,10 +143,127 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "069caa7c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
caratcutcolorclaritydepthtablepricexyz
00.23IdealESI261.555.03263.953.982.43
10.21PremiumESI159.861.03263.893.842.31
20.23GoodEVS156.965.03274.054.072.31
30.29PremiumIVS262.458.03344.204.232.63
40.31GoodJSI263.358.03354.344.352.75
\n", + "
" + ], + "text/plain": [ + " carat cut color clarity depth table price x y z\n", + "0 0.23 Ideal E SI2 61.5 55.0 326 3.95 3.98 2.43\n", + "1 0.21 Premium E SI1 59.8 61.0 326 3.89 3.84 2.31\n", + "2 0.23 Good E VS1 56.9 65.0 327 4.05 4.07 2.31\n", + "3 0.29 Premium I VS2 62.4 58.0 334 4.20 4.23 2.63\n", + "4 0.31 Good J SI2 63.3 58.0 335 4.34 4.35 2.75" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "diamonds = pd.read_csv(\n", " \"https://github.com/mwaskom/seaborn-data/raw/master/diamonds.csv\"\n", @@ -131,10 +289,135 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "97900f58", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(diamonds, aes(x=\"carat\")) + geom_histogram(binwidth=0.5))" ] @@ -172,10 +455,135 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "20d75550", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "smaller_diamonds = diamonds.query(\"carat < 3\").copy()\n", "\n", @@ -225,10 +633,135 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "d9d7e995", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(diamonds, aes(x=\"y\")) + geom_histogram(binwidth=0.5))" ] @@ -244,10 +777,140 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "ea8f8bf3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(diamonds, aes(x=\"y\"))\n", @@ -269,10 +932,123 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "e81ffb55", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
xyzprice
119630.000.00.005139
159510.000.00.006381
240678.0958.98.0612210
245200.000.00.0012800
262430.000.00.0015686
274290.000.00.0018034
491895.1531.85.122075
495560.000.00.002130
495570.000.00.002130
\n", + "
" + ], + "text/plain": [ + " x y z price\n", + "11963 0.00 0.0 0.00 5139\n", + "15951 0.00 0.0 0.00 6381\n", + "24067 8.09 58.9 8.06 12210\n", + "24520 0.00 0.0 0.00 12800\n", + "26243 0.00 0.0 0.00 15686\n", + "27429 0.00 0.0 0.00 18034\n", + "49189 5.15 31.8 5.12 2075\n", + "49556 0.00 0.0 0.00 2130\n", + "49557 0.00 0.0 0.00 2130" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "unusual = diamonds.query(\"y < 3 or y > 20\").loc[:, [\"x\", \"y\", \"z\", \"price\"]]\n", "unusual" @@ -344,7 +1120,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "ecf345a7", "metadata": {}, "outputs": [], @@ -364,10 +1140,135 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "15a43255", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(diamonds2, aes(x=\"x\", y=\"y\")) + geom_point())" ] @@ -385,10 +1286,195 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "0a4ea922", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
0201311517.05152.0830.081911.0UA1545N14228EWRIAH227.014005152013-01-01T10:00:00Z
1201311533.05294.0850.083020.0UA1714N24211LGAIAH227.014165292013-01-01T10:00:00Z
2201311542.05402.0923.085033.0AA1141N619AAJFKMIA160.010895402013-01-01T10:00:00Z
3201311544.0545-1.01004.01022-18.0B6725N804JBJFKBQN183.015765452013-01-01T10:00:00Z
4201311554.0600-6.0812.0837-25.0DL461N668DNLGAATL116.0762602013-01-01T11:00:00Z
\n", + "
" + ], + "text/plain": [ + " year month day dep_time sched_dep_time dep_delay arr_time \\\n", + "0 2013 1 1 517.0 515 2.0 830.0 \n", + "1 2013 1 1 533.0 529 4.0 850.0 \n", + "2 2013 1 1 542.0 540 2.0 923.0 \n", + "3 2013 1 1 544.0 545 -1.0 1004.0 \n", + "4 2013 1 1 554.0 600 -6.0 812.0 \n", + "\n", + " sched_arr_time arr_delay carrier flight tailnum origin dest air_time \\\n", + "0 819 11.0 UA 1545 N14228 EWR IAH 227.0 \n", + "1 830 20.0 UA 1714 N24211 LGA IAH 227.0 \n", + "2 850 33.0 AA 1141 N619AA JFK MIA 160.0 \n", + "3 1022 -18.0 B6 725 N804JB JFK BQN 183.0 \n", + "4 837 -25.0 DL 461 N668DN LGA ATL 116.0 \n", + "\n", + " distance hour minute time_hour \n", + "0 1400 5 15 2013-01-01T10:00:00Z \n", + "1 1416 5 29 2013-01-01T10:00:00Z \n", + "2 1089 5 40 2013-01-01T10:00:00Z \n", + "3 1576 5 45 2013-01-01T10:00:00Z \n", + "4 762 6 0 2013-01-01T11:00:00Z " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "url = \"https://raw.githubusercontent.com/byuidatascience/data4python4ds/master/data-raw/flights/flights.csv\"\n", "flights = pd.read_csv(url)\n", @@ -397,10 +1483,164 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "6849f4d9", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights2 = flights.assign(\n", " cancelled=lambda x: pd.isna(x[\"dep_time\"]),\n", @@ -452,10 +1692,138 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "e1719d8f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(diamonds, aes(x=\"price\"))\n", @@ -476,10 +1844,140 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "9388e24b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(diamonds, aes(x=\"price\"))\n", @@ -499,10 +1997,163 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "a3f333a6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(diamonds, aes(x=\"cut\", y=\"price\")) + geom_boxplot())" ] @@ -521,10 +2172,150 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "6949db81", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "mpg = pd.read_csv(\n", " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/mpg.csv\", index_col=0\n", @@ -544,10 +2335,168 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "a5b1ed09", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg) + geom_boxplot(aes(as_discrete(\"class\", order_by=\"..middle..\"), \"hwy\")))" ] @@ -562,10 +2511,172 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "920a4268", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg)\n", @@ -608,7 +2719,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "68d330d2", "metadata": {}, "outputs": [], @@ -630,10 +2741,112 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "e858cd22", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(ct_cut_color, aes(x=\"color\", y=\"cut\")) + geom_tile(aes(fill=\"value\")))" ] @@ -667,10 +2880,135 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "2afe2535", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(smaller_diamonds, aes(x=\"carat\", y=\"price\")) + geom_point())" ] @@ -688,10 +3026,136 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "b55707a9", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(smaller_diamonds, aes(x=\"carat\", y=\"price\")) + geom_point(alpha=1 / 20))" ] @@ -720,10 +3184,152 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "13079065", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
caratdepthtablepricexyz
count53940.00000053940.00000053940.00000053940.00000053940.00000053940.00000053940.000000
mean0.79794061.74940557.4571843932.7997225.7311575.7345263.538734
std0.4740111.4326212.2344913989.4397381.1217611.1421350.705699
min0.20000043.00000043.000000326.0000000.0000000.0000000.000000
25%0.40000061.00000056.000000950.0000004.7100004.7200002.910000
50%0.70000061.80000057.0000002401.0000005.7000005.7100003.530000
75%1.04000062.50000059.0000005324.2500006.5400006.5400004.040000
max5.01000079.00000095.00000018823.00000010.74000058.90000031.800000
\n", + "
" + ], + "text/plain": [ + " carat depth table price x \\\n", + "count 53940.000000 53940.000000 53940.000000 53940.000000 53940.000000 \n", + "mean 0.797940 61.749405 57.457184 3932.799722 5.731157 \n", + "std 0.474011 1.432621 2.234491 3989.439738 1.121761 \n", + "min 0.200000 43.000000 43.000000 326.000000 0.000000 \n", + "25% 0.400000 61.000000 56.000000 950.000000 4.710000 \n", + "50% 0.700000 61.800000 57.000000 2401.000000 5.700000 \n", + "75% 1.040000 62.500000 59.000000 5324.250000 6.540000 \n", + "max 5.010000 79.000000 95.000000 18823.000000 10.740000 \n", + "\n", + " y z \n", + "count 53940.000000 53940.000000 \n", + "mean 5.734526 3.538734 \n", + "std 1.142135 0.705699 \n", + "min 0.000000 0.000000 \n", + "25% 4.720000 2.910000 \n", + "50% 5.710000 3.530000 \n", + "75% 6.540000 4.040000 \n", + "max 58.900000 31.800000 " + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "diamonds.describe()" ] @@ -738,10 +3344,142 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "b4144440", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
caratdepthtablepricexyz
count53940.053940.053940.053940.053940.053940.053940.0
mean0.861.757.53932.85.75.73.5
std0.51.42.23989.41.11.10.7
min0.243.043.0326.00.00.00.0
25%0.461.056.0950.04.74.72.9
50%0.761.857.02401.05.75.73.5
75%1.062.559.05324.26.56.54.0
max5.079.095.018823.010.758.931.8
\n", + "
" + ], + "text/plain": [ + " carat depth table price x y z\n", + "count 53940.0 53940.0 53940.0 53940.0 53940.0 53940.0 53940.0\n", + "mean 0.8 61.7 57.5 3932.8 5.7 5.7 3.5\n", + "std 0.5 1.4 2.2 3989.4 1.1 1.1 0.7\n", + "min 0.2 43.0 43.0 326.0 0.0 0.0 0.0\n", + "25% 0.4 61.0 56.0 950.0 4.7 4.7 2.9\n", + "50% 0.7 61.8 57.0 2401.0 5.7 5.7 3.5\n", + "75% 1.0 62.5 59.0 5324.2 6.5 6.5 4.0\n", + "max 5.0 79.0 95.0 18823.0 10.7 58.9 31.8" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sum_table = diamonds.describe().round(1)\n", "sum_table" @@ -757,10 +3495,139 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "id": "cd2f8772", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
countmeanstdmin25%50%75%max
carat53940.00.80.50.20.40.71.05.0
depth53940.061.71.443.061.061.862.579.0
table53940.057.52.243.056.057.059.095.0
price53940.03932.83989.4326.0950.02401.05324.218823.0
x53940.05.71.10.04.75.76.510.7
y53940.05.71.10.04.75.76.558.9
z53940.03.50.70.02.93.54.031.8
\n", + "
" + ], + "text/plain": [ + " count mean std min 25% 50% 75% max\n", + "carat 53940.0 0.8 0.5 0.2 0.4 0.7 1.0 5.0\n", + "depth 53940.0 61.7 1.4 43.0 61.0 61.8 62.5 79.0\n", + "table 53940.0 57.5 2.2 43.0 56.0 57.0 59.0 95.0\n", + "price 53940.0 3932.8 3989.4 326.0 950.0 2401.0 5324.2 18823.0\n", + "x 53940.0 5.7 1.1 0.0 4.7 5.7 6.5 10.7\n", + "y 53940.0 5.7 1.1 0.0 4.7 5.7 6.5 58.9\n", + "z 53940.0 3.5 0.7 0.0 2.9 3.5 4.0 31.8" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sum_table = sum_table.T\n", "sum_table" @@ -780,10 +3647,110 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "id": "5afcacbc", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_58745/1534868389.py:2: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n", + " diamonds.groupby([\"cut\", \"color\"])[\"price\"]\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Sale price (thousands)
colorDEFGHIJ
cut       
Fair4.293.683.834.245.144.694.98
Good3.413.423.504.124.285.084.57
Very Good3.473.213.783.874.545.265.10
Premium3.633.544.324.505.225.956.29
Ideal2.632.603.373.723.894.454.92
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " diamonds.groupby([\"cut\", \"color\"])[\"price\"]\n", @@ -810,10 +3777,203 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "21e65189", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cutFairGoodVery GoodPremiumIdeal
color     
D163662151316032834
E224933240023373903
F312909216423313826
G314871229929244884
H303702182423603115
I175522120414282093
J119307678808896
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pd.crosstab(diamonds[\"color\"], diamonds[\"cut\"]).style.background_gradient(cmap=\"plasma\")" ] @@ -836,10 +3996,223 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "bb0162ba", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cutFairGoodVery GoodPremiumIdeal
color     
D163662151316032834
E224933240023373903
F312909216423313826
G314871229929244884
H303702182423603115
I175522120414282093
J119307678808896
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " pd.crosstab(diamonds[\"color\"], diamonds[\"cut\"])\n", @@ -858,10 +4231,106 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "id": "5d19072c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cutFairGoodVery GoodPremiumIdeal
color     
D163662151316032834
E224933240023373903
F312909216423313826
G314871229929244884
H303702182423603115
I175522120414282093
J119307678808896
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pd.crosstab(diamonds[\"color\"], diamonds[\"cut\"]).style.highlight_max().format(\"{:.0f}\")" ] @@ -886,10 +4355,165 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "id": "b479d5b1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pickupdropoffpassengersdistancefaretiptollstotalcolorpaymentpickup_zonedropoff_zonepickup_boroughdropoff_borough
02019-03-23 20:21:092019-03-23 20:27:2411.607.02.150.012.95yellowcredit cardLenox Hill WestUN/Turtle Bay SouthManhattanManhattan
12019-03-04 16:11:552019-03-04 16:19:0010.795.00.000.09.30yellowcashUpper West Side SouthUpper West Side SouthManhattanManhattan
22019-03-27 17:53:012019-03-27 18:00:2511.377.52.360.014.16yellowcredit cardAlphabet CityWest VillageManhattanManhattan
32019-03-10 01:23:592019-03-10 01:49:5117.7027.06.150.036.95yellowcredit cardHudson SqYorkville WestManhattanManhattan
42019-03-30 13:27:422019-03-30 13:37:1432.169.01.100.013.40yellowcredit cardMidtown EastYorkville WestManhattanManhattan
\n", + "
" + ], + "text/plain": [ + " pickup dropoff passengers distance fare tip \\\n", + "0 2019-03-23 20:21:09 2019-03-23 20:27:24 1 1.60 7.0 2.15 \n", + "1 2019-03-04 16:11:55 2019-03-04 16:19:00 1 0.79 5.0 0.00 \n", + "2 2019-03-27 17:53:01 2019-03-27 18:00:25 1 1.37 7.5 2.36 \n", + "3 2019-03-10 01:23:59 2019-03-10 01:49:51 1 7.70 27.0 6.15 \n", + "4 2019-03-30 13:27:42 2019-03-30 13:37:14 3 2.16 9.0 1.10 \n", + "\n", + " tolls total color payment pickup_zone \\\n", + "0 0.0 12.95 yellow credit card Lenox Hill West \n", + "1 0.0 9.30 yellow cash Upper West Side South \n", + "2 0.0 14.16 yellow credit card Alphabet City \n", + "3 0.0 36.95 yellow credit card Hudson Sq \n", + "4 0.0 13.40 yellow credit card Midtown East \n", + "\n", + " dropoff_zone pickup_borough dropoff_borough \n", + "0 UN/Turtle Bay South Manhattan Manhattan \n", + "1 Upper West Side South Manhattan Manhattan \n", + "2 West Village Manhattan Manhattan \n", + "3 Yorkville West Manhattan Manhattan \n", + "4 Yorkville West Manhattan Manhattan " + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "taxis = pd.read_csv(\"https://github.com/mwaskom/seaborn-data/raw/master/taxis.csv\")\n", "# turn the pickup time column into a datetime\n", @@ -912,20 +4536,1289 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "id": "ee971c9c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 6433 entries, 0 to 6432\n", + "Data columns (total 14 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 pickup 6433 non-null datetime64[ns]\n", + " 1 dropoff 6433 non-null datetime64[ns]\n", + " 2 passengers 6433 non-null int64 \n", + " 3 distance 6433 non-null float64 \n", + " 4 fare 6433 non-null float64 \n", + " 5 tip 6433 non-null float64 \n", + " 6 tolls 6433 non-null float64 \n", + " 7 total 6433 non-null float64 \n", + " 8 color 6433 non-null category \n", + " 9 payment 6389 non-null category \n", + " 10 pickup_zone 6407 non-null string \n", + " 11 dropoff_zone 6388 non-null string \n", + " 12 pickup_borough 6407 non-null category \n", + " 13 dropoff_borough 6388 non-null category \n", + "dtypes: category(4), datetime64[ns](2), float64(5), int64(1), string(2)\n", + "memory usage: 528.5 KB\n" + ] + } + ], "source": [ "taxis.info()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "id": "2015b1dc", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:34:43.401430\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "(\n", " taxis.set_index(\"pickup\")\n", @@ -949,10 +5842,1476 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "id": "51e86185", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:34:43.501760\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "(\n", " taxis.set_index(\"pickup\")\n", @@ -985,10 +7344,888 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "id": "79ceca92", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:34:43.567782\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "taxis.value_counts(\"payment\").sort_index().plot.bar(title=\"Counts\", rot=0);" ] @@ -1003,10 +8240,1011 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "id": "5efc5817", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:34:43.648471\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "taxis[\"tip\"].plot.hist(bins=30, title=\"Tip\");" ] @@ -1021,10 +9259,2055 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "id": "0b735d15", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:34:43.718910\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "(taxis[[\"fare\", \"tolls\", \"tip\"]].plot.box());" ] @@ -1039,10 +11322,7210 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "id": "66adada2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:34:43.811014\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "taxis.plot.scatter(x=\"fare\", y=\"tip\", alpha=0.7, ylim=(0, None));" ] @@ -1071,10 +18554,132 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "id": "32796b5f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭──────────────────────────────────────────────── skimpy summary ─────────────────────────────────────────────────╮\n",
+       "│          Data Summary                Data Types               Categories                                        │\n",
+       "│ ┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┓ ┏━━━━━━━━━━━━━┳━━━━━━━┓ ┏━━━━━━━━━━━━━━━━━━━━━━━┓                                │\n",
+       "│ ┃ Dataframe         ┃ Values ┃ ┃ Column Type ┃ Count ┃ ┃ Categorical Variables ┃                                │\n",
+       "│ ┡━━━━━━━━━━━━━━━━━━━╇━━━━━━━━┩ ┡━━━━━━━━━━━━━╇━━━━━━━┩ ┡━━━━━━━━━━━━━━━━━━━━━━━┩                                │\n",
+       "│ │ Number of rows    │ 6433   │ │ float64     │ 5     │ │ color                 │                                │\n",
+       "│ │ Number of columns │ 14     │ │ category    │ 4     │ │ payment               │                                │\n",
+       "│ └───────────────────┴────────┘ │ datetime64  │ 2     │ │ pickup_borough        │                                │\n",
+       "│                                │ string      │ 2     │ │ dropoff_borough       │                                │\n",
+       "│                                │ int64       │ 1     │ └───────────────────────┘                                │\n",
+       "│                                └─────────────┴───────┘                                                          │\n",
+       "│                                                     number                                                      │\n",
+       "│ ┏━━━━━━━━━━━━━━━━┳━━━━━━┳━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━┳━━━━━━━━━┓  │\n",
+       "│ ┃ column         ┃ NA   ┃ NA %   ┃ mean      ┃ sd      ┃ p0    ┃ p25    ┃ p50     ┃ p75   ┃ p100   ┃ hist    ┃  │\n",
+       "│ ┡━━━━━━━━━━━━━━━━╇━━━━━━╇━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━╇━━━━━━━━╇━━━━━━━━━╇━━━━━━━╇━━━━━━━━╇━━━━━━━━━┩  │\n",
+       "│ │ passengers     │    0 │      0 │     1.539 │   1.204 │     0 │      1 │       1 │     2 │      6 │  █▁  ▁  │  │\n",
+       "│ │ distance       │    0 │      0 │     3.025 │   3.828 │     0 │   0.98 │    1.64 │  3.21 │   36.7 │   █▁    │  │\n",
+       "│ │ fare           │    0 │      0 │     13.09 │   11.55 │     1 │    6.5 │     9.5 │    15 │    150 │   █▁    │  │\n",
+       "│ │ tip            │    0 │      0 │     1.979 │   2.449 │     0 │      0 │     1.7 │   2.8 │   33.2 │    █    │  │\n",
+       "│ │ tolls          │    0 │      0 │    0.3253 │   1.415 │     0 │      0 │       0 │     0 │  24.02 │    █    │  │\n",
+       "│ │ total          │    0 │      0 │     18.52 │   13.82 │   1.3 │   10.8 │   14.16 │  20.3 │  174.8 │   █▁    │  │\n",
+       "│ └────────────────┴──────┴────────┴───────────┴─────────┴───────┴────────┴─────────┴───────┴────────┴─────────┘  │\n",
+       "│                                                    category                                                     │\n",
+       "│ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓  │\n",
+       "│ ┃ column                        ┃ NA     ┃ NA %                               ┃ ordered        ┃ unique      ┃  │\n",
+       "│ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩  │\n",
+       "│ │ color                         │      0 │                                  0 │ False          │           2 │  │\n",
+       "│ │ payment                       │     44 │                 0.6839732628633608 │ False          │           3 │  │\n",
+       "│ │ pickup_borough                │     26 │                 0.4041660189647132 │ False          │           5 │  │\n",
+       "│ │ dropoff_borough               │     45 │                  0.699518109746619 │ False          │           6 │  │\n",
+       "│ └───────────────────────────────┴────────┴────────────────────────────────────┴────────────────┴─────────────┘  │\n",
+       "│                                                    datetime                                                     │\n",
+       "│ ┏━━━━━━━━━━━━━┳━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓  │\n",
+       "│ ┃ column      ┃ NA   ┃ NA %    ┃ first                        ┃ last                         ┃ frequency     ┃  │\n",
+       "│ ┡━━━━━━━━━━━━━╇━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩  │\n",
+       "│ │ pickup      │    0 │       0 │     2019-02-28 23:29:03      │     2019-03-31 23:43:45      │ None          │  │\n",
+       "│ │ dropoff     │    0 │       0 │     2019-02-28 23:32:35      │     2019-04-01 00:13:58      │ None          │  │\n",
+       "│ └─────────────┴──────┴─────────┴──────────────────────────────┴──────────────────────────────┴───────────────┘  │\n",
+       "│                                                     string                                                      │\n",
+       "│ ┏━━━━━━━━━━━┳━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┓  │\n",
+       "│ ┃           ┃    ┃           ┃          ┃           ┃          ┃           ┃ chars    ┃ words per ┃ total    ┃  │\n",
+       "│ ┃ column    ┃ NA ┃ NA %      ┃ shortest ┃ longest   ┃ min      ┃ max       ┃ per row  ┃ row       ┃ words    ┃  │\n",
+       "│ ┡━━━━━━━━━━━╇━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━┩  │\n",
+       "│ │ pickup_zo │ 26 │ 0.4041660 │ SoHo     │ Riverdale │ Allerton │ Yorkville │     16.2 │       2.5 │    15791 │  │\n",
+       "│ │ ne        │    │ 189647132 │          │ /North    │ /Pelham  │ West      │          │           │          │  │\n",
+       "│ │           │    │           │          │ Riverdale │ Gardens  │           │          │           │          │  │\n",
+       "│ │           │    │           │          │ /Fieldsto │          │           │          │           │          │  │\n",
+       "│ │           │    │           │          │ n         │          │           │          │           │          │  │\n",
+       "│ │ dropoff_z │ 45 │ 0.6995181 │ SoHo     │ Riverdale │ Allerton │ Yorkville │     16.3 │       2.5 │    15851 │  │\n",
+       "│ │ one       │    │  09746619 │          │ /North    │ /Pelham  │ West      │          │           │          │  │\n",
+       "│ │           │    │           │          │ Riverdale │ Gardens  │           │          │           │          │  │\n",
+       "│ │           │    │           │          │ /Fieldsto │          │           │          │           │          │  │\n",
+       "│ │           │    │           │          │ n         │          │           │          │           │          │  │\n",
+       "│ └───────────┴────┴───────────┴──────────┴───────────┴──────────┴───────────┴──────────┴───────────┴──────────┘  │\n",
+       "╰────────────────────────────────────────────────────── End ──────────────────────────────────────────────────────╯\n",
+       "
\n" + ], + "text/plain": [ + "╭──────────────────────────────────────────────── skimpy summary ─────────────────────────────────────────────────╮\n", + "│ \u001b[3m Data Summary \u001b[0m \u001b[3m Data Types \u001b[0m \u001b[3m Categories \u001b[0m │\n", + "│ ┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┓ ┏━━━━━━━━━━━━━┳━━━━━━━┓ ┏━━━━━━━━━━━━━━━━━━━━━━━┓ │\n", + "│ ┃\u001b[1;36m \u001b[0m\u001b[1;36mDataframe \u001b[0m\u001b[1;36m \u001b[0m┃\u001b[1;36m \u001b[0m\u001b[1;36mValues\u001b[0m\u001b[1;36m \u001b[0m┃ ┃\u001b[1;36m \u001b[0m\u001b[1;36mColumn Type\u001b[0m\u001b[1;36m \u001b[0m┃\u001b[1;36m \u001b[0m\u001b[1;36mCount\u001b[0m\u001b[1;36m \u001b[0m┃ ┃\u001b[1;36m \u001b[0m\u001b[1;36mCategorical Variables\u001b[0m\u001b[1;36m \u001b[0m┃ │\n", + "│ ┡━━━━━━━━━━━━━━━━━━━╇━━━━━━━━┩ ┡━━━━━━━━━━━━━╇━━━━━━━┩ ┡━━━━━━━━━━━━━━━━━━━━━━━┩ │\n", + "│ │ Number of rows │ 6433 │ │ float64 │ 5 │ │ color │ │\n", + "│ │ Number of columns │ 14 │ │ category │ 4 │ │ payment │ │\n", + "│ └───────────────────┴────────┘ │ datetime64 │ 2 │ │ pickup_borough │ │\n", + "│ │ string │ 2 │ │ dropoff_borough │ │\n", + "│ │ int64 │ 1 │ └───────────────────────┘ │\n", + "│ └─────────────┴───────┘ │\n", + "│ \u001b[3m number \u001b[0m │\n", + "│ ┏━━━━━━━━━━━━━━━━┳━━━━━━┳━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━┳━━━━━━━━━┓ │\n", + "│ ┃\u001b[1m \u001b[0m\u001b[1mcolumn \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA % \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mmean \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1msd \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mp0 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mp25 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mp50 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mp75 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mp100 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mhist \u001b[0m\u001b[1m \u001b[0m┃ │\n", + "│ ┡━━━━━━━━━━━━━━━━╇━━━━━━╇━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━╇━━━━━━━━╇━━━━━━━━━╇━━━━━━━╇━━━━━━━━╇━━━━━━━━━┩ │\n", + "│ │ \u001b[38;5;141mpassengers \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 1.539\u001b[0m │ \u001b[36m 1.204\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 1\u001b[0m │ \u001b[36m 1\u001b[0m │ \u001b[36m 2\u001b[0m │ \u001b[36m 6\u001b[0m │ \u001b[32m █▁ ▁ \u001b[0m │ │\n", + "│ │ \u001b[38;5;141mdistance \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 3.025\u001b[0m │ \u001b[36m 3.828\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0.98\u001b[0m │ \u001b[36m 1.64\u001b[0m │ \u001b[36m 3.21\u001b[0m │ \u001b[36m 36.7\u001b[0m │ \u001b[32m █▁ \u001b[0m │ │\n", + "│ │ \u001b[38;5;141mfare \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 13.09\u001b[0m │ \u001b[36m 11.55\u001b[0m │ \u001b[36m 1\u001b[0m │ \u001b[36m 6.5\u001b[0m │ \u001b[36m 9.5\u001b[0m │ \u001b[36m 15\u001b[0m │ \u001b[36m 150\u001b[0m │ \u001b[32m █▁ \u001b[0m │ │\n", + "│ │ \u001b[38;5;141mtip \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 1.979\u001b[0m │ \u001b[36m 2.449\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 1.7\u001b[0m │ \u001b[36m 2.8\u001b[0m │ \u001b[36m 33.2\u001b[0m │ \u001b[32m █ \u001b[0m │ │\n", + "│ │ \u001b[38;5;141mtolls \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0.3253\u001b[0m │ \u001b[36m 1.415\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 24.02\u001b[0m │ \u001b[32m █ \u001b[0m │ │\n", + "│ │ \u001b[38;5;141mtotal \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 18.52\u001b[0m │ \u001b[36m 13.82\u001b[0m │ \u001b[36m 1.3\u001b[0m │ \u001b[36m 10.8\u001b[0m │ \u001b[36m 14.16\u001b[0m │ \u001b[36m 20.3\u001b[0m │ \u001b[36m 174.8\u001b[0m │ \u001b[32m █▁ \u001b[0m │ │\n", + "│ └────────────────┴──────┴────────┴───────────┴─────────┴───────┴────────┴─────────┴───────┴────────┴─────────┘ │\n", + "│ \u001b[3m category \u001b[0m │\n", + "│ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓ │\n", + "│ ┃\u001b[1m \u001b[0m\u001b[1mcolumn \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA % \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mordered \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1munique \u001b[0m\u001b[1m \u001b[0m┃ │\n", + "│ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩ │\n", + "│ │ \u001b[38;5;141mcolor \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[38;5;45mFalse \u001b[0m │ \u001b[36m 2\u001b[0m │ │\n", + "│ │ \u001b[38;5;141mpayment \u001b[0m │ \u001b[36m 44\u001b[0m │ \u001b[36m 0.6839732628633608\u001b[0m │ \u001b[38;5;45mFalse \u001b[0m │ \u001b[36m 3\u001b[0m │ │\n", + "│ │ \u001b[38;5;141mpickup_borough \u001b[0m │ \u001b[36m 26\u001b[0m │ \u001b[36m 0.4041660189647132\u001b[0m │ \u001b[38;5;45mFalse \u001b[0m │ \u001b[36m 5\u001b[0m │ │\n", + "│ │ \u001b[38;5;141mdropoff_borough \u001b[0m │ \u001b[36m 45\u001b[0m │ \u001b[36m 0.699518109746619\u001b[0m │ \u001b[38;5;45mFalse \u001b[0m │ \u001b[36m 6\u001b[0m │ │\n", + "│ └───────────────────────────────┴────────┴────────────────────────────────────┴────────────────┴─────────────┘ │\n", + "│ \u001b[3m datetime \u001b[0m │\n", + "│ ┏━━━━━━━━━━━━━┳━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓ │\n", + "│ ┃\u001b[1m \u001b[0m\u001b[1mcolumn \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA % \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mfirst \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mlast \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mfrequency \u001b[0m\u001b[1m \u001b[0m┃ │\n", + "│ ┡━━━━━━━━━━━━━╇━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩ │\n", + "│ │ \u001b[38;5;141mpickup \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[31m 2019-02-28 23:29:03 \u001b[0m │ \u001b[31m 2019-03-31 23:43:45 \u001b[0m │ \u001b[38;5;141mNone \u001b[0m │ │\n", + "│ │ \u001b[38;5;141mdropoff \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[31m 2019-02-28 23:32:35 \u001b[0m │ \u001b[31m 2019-04-01 00:13:58 \u001b[0m │ \u001b[38;5;141mNone \u001b[0m │ │\n", + "│ └─────────────┴──────┴─────────┴──────────────────────────────┴──────────────────────────────┴───────────────┘ │\n", + "│ \u001b[3m string \u001b[0m │\n", + "│ ┏━━━━━━━━━━━┳━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┓ │\n", + "│ ┃\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mchars \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mwords per\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mtotal \u001b[0m\u001b[1m \u001b[0m┃ │\n", + "│ ┃\u001b[1m \u001b[0m\u001b[1mcolumn \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA % \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mshortest\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mlongest \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mmin \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mmax \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mper row \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mrow \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mwords \u001b[0m\u001b[1m \u001b[0m┃ │\n", + "│ ┡━━━━━━━━━━━╇━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━┩ │\n", + "│ │ \u001b[38;5;141mpickup_zo\u001b[0m │ \u001b[36m26\u001b[0m │ \u001b[36m0.4041660\u001b[0m │ \u001b[38;5;141mSoHo \u001b[0m │ \u001b[38;5;141mRiverdale\u001b[0m │ \u001b[38;5;141mAllerton\u001b[0m │ \u001b[38;5;141mYorkville\u001b[0m │ \u001b[36m 16.2\u001b[0m │ \u001b[36m 2.5\u001b[0m │ \u001b[36m 15791\u001b[0m │ │\n", + "│ │ \u001b[38;5;141mne \u001b[0m │ │ \u001b[36m189647132\u001b[0m │ │ \u001b[38;5;141m/North \u001b[0m │ \u001b[38;5;141m/Pelham \u001b[0m │ \u001b[38;5;141mWest \u001b[0m │ │ │ │ │\n", + "│ │ │ │ │ │ \u001b[38;5;141mRiverdale\u001b[0m │ \u001b[38;5;141mGardens \u001b[0m │ │ │ │ │ │\n", + "│ │ │ │ │ │ \u001b[38;5;141m/Fieldsto\u001b[0m │ │ │ │ │ │ │\n", + "│ │ │ │ │ │ \u001b[38;5;141mn \u001b[0m │ │ │ │ │ │ │\n", + "│ │ \u001b[38;5;141mdropoff_z\u001b[0m │ \u001b[36m45\u001b[0m │ \u001b[36m0.6995181\u001b[0m │ \u001b[38;5;141mSoHo \u001b[0m │ \u001b[38;5;141mRiverdale\u001b[0m │ \u001b[38;5;141mAllerton\u001b[0m │ \u001b[38;5;141mYorkville\u001b[0m │ \u001b[36m 16.3\u001b[0m │ \u001b[36m 2.5\u001b[0m │ \u001b[36m 15851\u001b[0m │ │\n", + "│ │ \u001b[38;5;141mone \u001b[0m │ │ \u001b[36m 09746619\u001b[0m │ │ \u001b[38;5;141m/North \u001b[0m │ \u001b[38;5;141m/Pelham \u001b[0m │ \u001b[38;5;141mWest \u001b[0m │ │ │ │ │\n", + "│ │ │ │ │ │ \u001b[38;5;141mRiverdale\u001b[0m │ \u001b[38;5;141mGardens \u001b[0m │ │ │ │ │ │\n", + "│ │ │ │ │ │ \u001b[38;5;141m/Fieldsto\u001b[0m │ │ │ │ │ │ │\n", + "│ │ │ │ │ │ \u001b[38;5;141mn \u001b[0m │ │ │ │ │ │ │\n", + "│ └───────────┴────┴───────────┴──────────┴───────────┴──────────┴───────────┴──────────┴───────────┴──────────┘ │\n", + "╰────────────────────────────────────────────────────── End ──────────────────────────────────────────────────────╯\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "skim(taxis)" ] @@ -1119,7 +18724,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/functions.ipynb b/functions.ipynb index 7d55cbc..70c4fd3 100644 --- a/functions.ipynb +++ b/functions.ipynb @@ -48,10 +48,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "0450ad6c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hello Ada, and welcome!\n" + ] + } + ], "source": [ "def welcome_message(name):\n", " return f\"Hello {name}, and welcome!\"\n", @@ -73,10 +81,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "dd49bee5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "('Hello student', 4.8)\n", + "('Hello Ada', 4.8)\n" + ] + } + ], "source": [ "def score_message(score, name=\"student\"):\n", " \"\"\"This is a doc-string, a string describing a function.\n", @@ -121,10 +138,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "50dfff24", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function score_message in module __main__:\n", + "\n", + "score_message(score, name='student')\n", + " This is a doc-string, a string describing a function.\n", + " Args:\n", + " score (float): Raw score\n", + " name (str): Name of student\n", + " Returns:\n", + " str: A hello message.\n", + " float: A normalised score.\n", + "\n" + ] + } + ], "source": [ "help(score_message)" ] @@ -159,10 +194,47 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "43d10e40", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (10, 4)
abcd
f64f64f64f64
0.4345330.9990720.3288630.269584
0.00.4799171.00.676026
1.00.00.3837450.381636
0.6584960.4682960.0809160.283739
0.3495880.1646770.6419610.38563
0.3624860.3741330.4753370.024468
0.0426121.1409950.4003250.121255
0.5385220.3190620.5273250.292675
0.5442450.6919370.01.0
0.3524920.2954480.4701810.0
" + ], + "text/plain": [ + "shape: (10, 4)\n", + "┌──────────┬──────────┬──────────┬──────────┐\n", + "│ a ┆ b ┆ c ┆ d │\n", + "│ --- ┆ --- ┆ --- ┆ --- │\n", + "│ f64 ┆ f64 ┆ f64 ┆ f64 │\n", + "╞══════════╪══════════╪══════════╪══════════╡\n", + "│ 0.434533 ┆ 0.999072 ┆ 0.328863 ┆ 0.269584 │\n", + "│ 0.0 ┆ 0.479917 ┆ 1.0 ┆ 0.676026 │\n", + "│ 1.0 ┆ 0.0 ┆ 0.383745 ┆ 0.381636 │\n", + "│ 0.658496 ┆ 0.468296 ┆ 0.080916 ┆ 0.283739 │\n", + "│ 0.349588 ┆ 0.164677 ┆ 0.641961 ┆ 0.38563 │\n", + "│ 0.362486 ┆ 0.374133 ┆ 0.475337 ┆ 0.024468 │\n", + "│ 0.042612 ┆ 1.140995 ┆ 0.400325 ┆ 0.121255 │\n", + "│ 0.538522 ┆ 0.319062 ┆ 0.527325 ┆ 0.292675 │\n", + "│ 0.544245 ┆ 0.691937 ┆ 0.0 ┆ 1.0 │\n", + "│ 0.352492 ┆ 0.295448 ┆ 0.470181 ┆ 0.0 │\n", + "└──────────┴──────────┴──────────┴──────────┘" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import numpy as np\n", "import polars as pl\n", @@ -241,7 +313,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "406648b6", "metadata": {}, "outputs": [], @@ -275,20 +347,76 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "74834d90", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (3,)
f64
0.0
0.5
1.0
" + ], + "text/plain": [ + "shape: (3,)\n", + "Series: '' [f64]\n", + "[\n", + "\t0.0\n", + "\t0.5\n", + "\t1.0\n", + "]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "rescale(pl.Series([-10, 0, 10]))" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "8a0e643e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (5,)
f64
0.0
0.25
0.5
null
1.0
" + ], + "text/plain": [ + "shape: (5,)\n", + "Series: '' [f64]\n", + "[\n", + "\t0.0\n", + "\t0.25\n", + "\t0.5\n", + "\tnull\n", + "\t1.0\n", + "]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "rescale(pl.Series([1, 2, 3, None, 5]))" ] @@ -306,7 +434,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "ce066e4f", "metadata": {}, "outputs": [], @@ -334,10 +462,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "f8b80894", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (5,)
f64
0.0
0.0
0.0
NaN
0.0
" + ], + "text/plain": [ + "shape: (5,)\n", + "Series: '' [f64]\n", + "[\n", + "\t0.0\n", + "\t0.0\n", + "\t0.0\n", + "\tNaN\n", + "\t0.0\n", + "]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "rescale(pl.Series([1, 2, 3, float(\"inf\"), 5], strict=False))" ] @@ -352,10 +509,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "cd63ddea", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (5,)
f64
0.0
0.25
0.5
null
1.0
" + ], + "text/plain": [ + "shape: (5,)\n", + "Series: '' [f64]\n", + "[\n", + "\t0.0\n", + "\t0.25\n", + "\t0.5\n", + "\tnull\n", + "\t1.0\n", + "]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "def rescale(x):\n", " x = x.cast(pl.Float64).replace(float(\"inf\"), None)\n", @@ -441,10 +627,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "647da443", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hello World!\n" + ] + } + ], "source": [ "def var_func():\n", " str_variable = \"Hello World!\"\n", @@ -467,10 +661,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "c5fb049d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "y is inside a function: I'm a global variable\n", + "y is outside a function: I'm a global variable\n" + ] + } + ], "source": [ "y = \"I'm a global variable\"\n", "\n", @@ -506,7 +709,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/introduction.ipynb b/introduction.ipynb index 60bfa83..2a70fd3 100644 --- a/introduction.ipynb +++ b/introduction.ipynb @@ -17,7 +17,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "51a55374", "metadata": { "tags": [ @@ -33,14 +33,116 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "209ef434", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "cluster_0\n", + "\n", + "Understand\n", + "\n", + "\n", + "\n", + "Import\n", + "\n", + "Import\n", + "\n", + "\n", + "\n", + "Clean\n", + "\n", + "Clean\n", + "\n", + "\n", + "\n", + "Import->Clean\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Analyse\n", + "\n", + "Analyse\n", + "\n", + "\n", + "\n", + "Clean->Analyse\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Visualise\n", + "\n", + "Visualise\n", + "\n", + "\n", + "\n", + "Visualise->Analyse\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Transform\n", + "\n", + "Transform\n", + "\n", + "\n", + "\n", + "Analyse->Transform\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Communicate\n", + "\n", + "Communicate\n", + "\n", + "\n", + "\n", + "Analyse->Communicate\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Transform->Visualise\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# | echo: false\n", "import graphviz\n", @@ -138,14 +240,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "26faf349", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Compiled with Python version: 3.12.13 (main, Mar 3 2026, 12:39:30) [Clang 17.0.0 (clang-1700.6.3.2)]\n" + ] + } + ], "source": [ "# | echo: false\n", "import sys\n", @@ -187,7 +297,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/iteration.ipynb b/iteration.ipynb index fc1452a..74021f3 100644 --- a/iteration.ipynb +++ b/iteration.ipynb @@ -21,12 +21,12 @@ "\n", "One tool for reducing duplication is functions, which reduce duplication by identifying repeated patterns of code and extract them out into independent pieces that can be easily reused and updated. Another tool for reducing duplication is *iteration*, which helps you when you need to do the same thing to multiple inputs: repeating the same operation on different columns, or on different datasets.\n", "\n", - "In this chapter you'll learn about iteration in three ways: explicit iteration, using for loops and while loops; iteration via comprehensions (eg list comprehensions); and iteration for **pandas** data frames." + "In this chapter you'll learn about iteration in three ways: explicit iteration, using for loops and while loops; iteration via comprehensions (eg list comprehensions); and iteration for **polars** data frames." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "51a55374", "metadata": { "tags": [ @@ -51,7 +51,7 @@ "source": [ "### Prerequisites\n", "\n", - "This chapter will use the **pandas** data analysis package." + "This chapter will use the **polars** data analysis package." ] }, { @@ -68,10 +68,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "a2bbd41c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Lovelace\n", + "Smith\n", + "Pigou\n", + "Babbage\n" + ] + } + ], "source": [ "name_list = [\"Lovelace\", \"Smith\", \"Pigou\", \"Babbage\"]\n", "\n", @@ -106,10 +117,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "835ebda7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hello\n", + "---end entry---\n" + ] + } + ], "source": [ "for entry in [\"hello\"]:\n", " print(entry)\n", @@ -126,10 +146,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "2a19ac2e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "h\n", + "---end entry---\n", + "e\n", + "---end entry---\n", + "l\n", + "---end entry---\n", + "l\n", + "---end entry---\n", + "o\n", + "---end entry---\n" + ] + } + ], "source": [ "for entry in \"hello\":\n", " print(entry)\n", @@ -150,10 +187,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "239e133f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The name in position 0 is Lovelace\n", + "The name in position 1 is Smith\n", + "The name in position 2 is Hopper\n", + "The name in position 3 is Babbage\n" + ] + } + ], "source": [ "name_list = [\"Lovelace\", \"Smith\", \"Hopper\", \"Babbage\"]\n", "\n", @@ -171,10 +219,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "b66c5c53", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The name in position 1 is Lovelace\n", + "The name in position 2 is Smith\n", + "The name in position 3 is Hopper\n", + "The name in position 4 is Babbage\n" + ] + } + ], "source": [ "for i, name in enumerate(name_list, start=1):\n", " print(f\"The name in position {i} is {name}\")" @@ -190,10 +249,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "010239fe", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "In Paris, the temperature is 28 degrees C today.\n", + "In London, the temperature is 22 degrees C today.\n", + "In Seville, the temperature is 36 degrees C today.\n", + "In Wellesley, the temperature is 29 degrees C today.\n" + ] + } + ], "source": [ "cities_to_temps = {\"Paris\": 28, \"London\": 22, \"Seville\": 36, \"Wellesley\": 29}\n", "\n", @@ -217,10 +287,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "8ea3efc5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ada Lovelace\n", + "Adam Smith\n", + "Grace Hopper\n", + "Charles Babbage\n" + ] + } + ], "source": [ "first_names = [\"Ada\", \"Adam\", \"Grace\", \"Charles\"]\n", "last_names = [\"Lovelace\", \"Smith\", \"Hopper\", \"Babbage\"]\n", @@ -257,10 +338,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "7efed381", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[51, 52, 53, 54, 55, 56, 57, 58, 59, 60]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "num_list = range(50, 60)\n", "[1 + num for num in num_list]" @@ -286,10 +378,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "722fda21", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39]\n" + ] + } + ], "source": [ "number_list = range(1, 40)\n", "divide_list = [x for x in number_list if x % 3 == 0]\n", @@ -308,10 +408,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "b6e80d6b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['Adam Smith', 'leonara smith']\n" + ] + } + ], "source": [ "names_list = [\"Joe Bloggs\", \"Adam Smith\", \"Sandra Noone\", \"leonara smith\"]\n", "smith_list = [x for x in names_list if \"smith\" in x.lower()]\n", @@ -330,10 +438,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "f348bfb6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['Not Smith!', 'Adam Smith', 'Not Smith!', 'leonara smith']\n" + ] + } + ], "source": [ "names_list = [\"Joe Bloggs\", \"Adam Smith\", \"Sandra Noone\", \"leonara smith\"]\n", "smith_list = [x if \"smith\" in x.lower() else \"Not Smith!\" for x in names_list]\n", @@ -350,10 +466,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "74e4fcc7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['Ada Lovelace', 'Adam Smith', 'Grace Hopper', 'Charles Babbage']\n" + ] + } + ], "source": [ "first_names = [\"Ada\", \"Adam\", \"Grace\", \"Charles\"]\n", "last_names = [\"Lovelace\", \"Smith\", \"Hopper\", \"Babbage\"]\n", @@ -371,10 +495,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "2c82cf1f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[['Ada Lovelace', 'Adam Lovelace'], ['Ada Smith', 'Adam Smith']]\n" + ] + } + ], "source": [ "first_names = [\"Ada\", \"Adam\"]\n", "last_names = [\"Lovelace\", \"Smith\"]\n", @@ -394,10 +526,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "acef16ae", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'Ada': 'Lovelace', 'Adam': 'Smith'}" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "{key: value for key, value in zip(first_names, last_names)}" ] @@ -420,10 +563,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "3e47ba02", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10\n", + "9\n", + "8\n", + "7\n", + "6\n", + "5\n", + "4\n", + "3\n", + "2\n", + "1\n", + "execution complete\n" + ] + } + ], "source": [ "n = 10\n", "while n > 0:\n", @@ -452,11 +613,13 @@ "id": "5ec0643e", "metadata": {}, "source": [ - "## Iteration with **pandas** Data Frames\n", + "## Iteration with **polars** Data Frames\n", "\n", - "For loops, while loops, and comprehensions all work on **pandas** data frames, but they are generally a bad way to get things done because they are slow and not memory efficient. To aid cases where iteration is needed, **pandas** has built-in methods for iteration depending on what you need to do.\n", + "For loops, while loops, and comprehensions can be used with data frames, but in **Polars**, they are even more strongly discouraged than in pandas. **Polars** is built on a columnar, vectorized, and expression-based engine, so row-by-row iteration breaks performance and prevents optimizations.\n", "\n", "These built-in methods for iteration have an overlap with what we've seen in @sec-data-transform but we'll dig a little deeper into `assign()`/assignment operations, `apply()`, and `eval()` here.\n", + "\n", + "Instead of iterating, **Polars** encourages you to use expressions and lazy evaluation, which are much faster and more memory efficient.\n", "\n" ] }, @@ -474,15 +637,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "b3116809", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 4)
abcd
f64f64f64f64
-0.405242-1.3517780.4098180.975997
1.425904-1.610260.456314-0.371164
-0.555008-0.13544-0.3194540.612513
1.842544-1.7097670.0183770.304892
0.926347-1.3718410.3992051.433585
0.649944-0.011038-0.904321-0.5888
" + ], + "text/plain": [ + "shape: (6, 4)\n", + "┌───────────┬───────────┬───────────┬───────────┐\n", + "│ a ┆ b ┆ c ┆ d │\n", + "│ --- ┆ --- ┆ --- ┆ --- │\n", + "│ f64 ┆ f64 ┆ f64 ┆ f64 │\n", + "╞═══════════╪═══════════╪═══════════╪═══════════╡\n", + "│ -0.405242 ┆ -1.351778 ┆ 0.409818 ┆ 0.975997 │\n", + "│ 1.425904 ┆ -1.61026 ┆ 0.456314 ┆ -0.371164 │\n", + "│ -0.555008 ┆ -0.13544 ┆ -0.319454 ┆ 0.612513 │\n", + "│ 1.842544 ┆ -1.709767 ┆ 0.018377 ┆ 0.304892 │\n", + "│ 0.926347 ┆ -1.371841 ┆ 0.399205 ┆ 1.433585 │\n", + "│ 0.649944 ┆ -0.011038 ┆ -0.904321 ┆ -0.5888 │\n", + "└───────────┴───────────┴───────────┴───────────┘" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import numpy as np\n", - "import pandas as pd\n", + "import polars as pl\n", "\n", - "df = pd.DataFrame(np.random.normal(size=(6, 4)), columns=[\"a\", \"b\", \"c\", \"d\"])\n", + "df = pl.DataFrame(np.random.normal(size=(6, 4)), schema=[\"a\", \"b\", \"c\", \"d\"])\n", "df" ] }, @@ -491,27 +687,88 @@ "id": "0e826ad5", "metadata": {}, "source": [ - "**pandas** has many built-in functions that are already built to iterate over rows and columns; for example, to compute the median of rows or columns respectively:" + "**polars** has built-in expressions designed to operate over columns and rows. For example, to compute the median:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "ac909c2f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (1, 4)
abcd
f64f64f64f64
0.788146-1.361810.2087910.458703
" + ], + "text/plain": [ + "shape: (1, 4)\n", + "┌──────────┬──────────┬──────────┬──────────┐\n", + "│ a ┆ b ┆ c ┆ d │\n", + "│ --- ┆ --- ┆ --- ┆ --- │\n", + "│ f64 ┆ f64 ┆ f64 ┆ f64 │\n", + "╞══════════╪══════════╪══════════╪══════════╡\n", + "│ 0.788146 ┆ -1.36181 ┆ 0.208791 ┆ 0.458703 │\n", + "└──────────┴──────────┴──────────┴──────────┘" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "df.median(axis=\"rows\") # can also use axis=1" + "df.select(pl.all().median())" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "96426002", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 1)
row_median
f64
0.002288
0.042575
-0.227447
0.161635
0.662776
-0.299919
" + ], + "text/plain": [ + "shape: (6, 1)\n", + "┌────────────┐\n", + "│ row_median │\n", + "│ --- │\n", + "│ f64 │\n", + "╞════════════╡\n", + "│ 0.002288 │\n", + "│ 0.042575 │\n", + "│ -0.227447 │\n", + "│ 0.161635 │\n", + "│ 0.662776 │\n", + "│ -0.299919 │\n", + "└────────────┘" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "df.median(axis=\"columns\") # can also use axis=0" + "df.select(pl.concat_list(pl.all()).list.median().alias(\"row_median\"))" ] }, { @@ -524,10 +781,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "060b6815", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "291 μs ± 1.95 μs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)\n" + ] + } + ], "source": [ "# Do not do this!\n", "\n", @@ -535,7 +800,7 @@ "def add_five_slow(df):\n", " for i in range(len(df)):\n", " for j in range(len(df.columns)):\n", - " df.iloc[i, j] = df.iloc[i, j] + 5\n", + " df[i, j] = df[i, j] + 5\n", "\n", "\n", "%timeit add_five_slow(df)" @@ -546,15 +811,23 @@ "id": "8246132e", "metadata": {}, "source": [ - "But to do this, every individual cell must be accessed and operated on—so it is very slow, taking milliseconds. **pandas** has far faster ways of performing the same operation. For simple operations on data frames with consistent type, you can simply add five to the whole data frame:" + "But to do this, every individual cell must be accessed and operated on—so it is very slow, taking milliseconds. **polars** has far faster ways of performing the same operation. For simple operations on data frames with consistent type, you can simply add five to the whole data frame:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "1a48ae52", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "56.7 μs ± 601 ns per loop (mean ± std. dev. of 7 runs, 10,000 loops each)\n" + ] + } + ], "source": [ "%timeit df + 5" ] @@ -572,19 +845,19 @@ "id": "7313616e", "metadata": {}, "source": [ - "This also works on a per column basis, so you can do `df[\"a\"] = df[\"a\"] + 5` and so on.\n", + "This also works on a per column basis, so you can do `df.with_columns(pl.col(\"a\") + 5)` and so on.\n", "\n", - "These operations have equivalents using the `assign()` operator, which allows for *method chaining*; stringing multiple operations together. The `assign()` operator version of `df[\"new_a\"] = df[\"a\"] + 5` would be" + "These operations have equivalents using method chaining; stringing multiple operations together. The version of `df.with_columns(new_a = pl.col(\"a\") + 5)` would be:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "f7391dc5", "metadata": {}, "outputs": [], "source": [ - "df = df.assign(new_a=lambda x: x[\"a\"] + 5)" + "df = df.with_columns(new_a=pl.col(\"a\") + 5)" ] }, { @@ -592,19 +865,63 @@ "id": "76aec162", "metadata": {}, "source": [ - "### Apply\n", + "### Expressions (Polars' Alternative to apply)\n", + "\n", + "What happens if you have a more complicated operation you want to perform? In pandas, you might reach for `apply()`. In **polars**, you almost never need an equivalent because its expression API is incredibly expressive.\n", "\n", - "What happens if you have a more complicated function you want to iterate over? This is where **pandas**' `apply()` comes in, and can be used with assignment. `apply()` can also be used across rows or columns. Like `assign()`, it can be combined with a lambda function and used with either the whole data frame or just a column (in which case no need to specify `axis=`)." + "Most \"complicated\" operations can be expressed directly using **polars'** built-in expressions:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "31adcb3f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 6)
abcdnew_aresult
f64f64f64f64f64f64
40554.59475840553.64822240555.40981840555.97599740559.594758-7.814557
40556.42590440553.3897440555.45631440554.62883640561.425904-6.288454
40554.44499240554.8645640554.68054640555.61251340559.444992-6.018383
40556.84254440553.29023340555.01837740555.30489240561.842544-5.533329
40555.92634740553.62815940555.39920540556.43358540560.926347-6.492421
40555.64994440554.98896240554.09567940554.411240560.649944-4.104063
" + ], + "text/plain": [ + "shape: (6, 6)\n", + "┌──────────────┬──────────────┬──────────────┬──────────────┬──────────────┬───────────┐\n", + "│ a ┆ b ┆ c ┆ d ┆ new_a ┆ result │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │\n", + "╞══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪═══════════╡\n", + "│ 40554.594758 ┆ 40553.648222 ┆ 40555.409818 ┆ 40555.975997 ┆ 40559.594758 ┆ -7.814557 │\n", + "│ 40556.425904 ┆ 40553.38974 ┆ 40555.456314 ┆ 40554.628836 ┆ 40561.425904 ┆ -6.288454 │\n", + "│ 40554.444992 ┆ 40554.86456 ┆ 40554.680546 ┆ 40555.612513 ┆ 40559.444992 ┆ -6.018383 │\n", + "│ 40556.842544 ┆ 40553.290233 ┆ 40555.018377 ┆ 40555.304892 ┆ 40561.842544 ┆ -5.533329 │\n", + "│ 40555.926347 ┆ 40553.628159 ┆ 40555.399205 ┆ 40556.433585 ┆ 40560.926347 ┆ -6.492421 │\n", + "│ 40555.649944 ┆ 40554.988962 ┆ 40554.095679 ┆ 40554.4112 ┆ 40560.649944 ┆ -4.104063 │\n", + "└──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴───────────┘" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "df.apply(lambda x: x[\"a\"] - x[\"new_a\"].mean() * x[\"c\"] / x[\"b\"], axis=1)" + "# Don't do this (slow, row-wise)\n", + "mean_new_a = df.select(pl.col(\"new_a\").mean()).item()\n", + "df.with_columns(\n", + " result=pl.struct([\"a\", \"b\", \"c\"]).map_elements(\n", + " lambda x: x[\"a\"] - mean_new_a * x[\"c\"] / x[\"b\"], return_dtype=pl.Float64\n", + " )\n", + ")\n", + "\n", + "# Do this instead (fast, vectorized)\n", + "df.with_columns(result=pl.col(\"a\") - pl.col(\"new_a\").mean() * pl.col(\"c\") / pl.col(\"b\"))" ] }, { @@ -612,24 +929,7 @@ "id": "78b558f4", "metadata": {}, "source": [ - "Note that this is just an example: you could still do this entire operation without using apply! But you will sometimes find yourself with cases where you do need to use it.\n", - "\n", - "Apply also works with functions, including user-defined functions:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "136d435d", - "metadata": {}, - "outputs": [], - "source": [ - "def complicated_function(x):\n", - " return x - x.mean()\n", - "\n", - "\n", - "df = df.apply(complicated_function, axis=1)\n", - "df" + "The first expression would work, but it evaluates the computation row by row using a **python lambda**, which is slow and prevents **polars** from optimizing the query. The second approach uses native expressions, allowing **polars** to execute the computation efficiently in a fully vectorized and optimized manner." ] }, { @@ -637,19 +937,50 @@ "id": "171be2c9", "metadata": {}, "source": [ - "### Eval(uate)\n", - "\n", - "`eval()` evaluates a string describing operations on DataFrame columns to create new columns. It operates on columns only, not rows or elements. Here's an example:" + "In **polars**, there's no `eval()` — you use expressions directly instead:\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "8d9defd3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 6)
abcdnew_aratio
f64f64f64f64f64f64
40554.59475840553.64822240555.40981840555.97599740559.5947580.999877
40556.42590440553.3897440555.45631440554.62883640561.4259040.999877
40554.44499240554.8645640554.68054640555.61251340559.4449920.999877
40556.84254440553.29023340555.01837740555.30489240561.8425440.999877
40555.92634740553.62815940555.39920540556.43358540560.9263470.999877
40555.64994440554.98896240554.09567940554.411240560.6499440.999877
" + ], + "text/plain": [ + "shape: (6, 6)\n", + "┌──────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────┐\n", + "│ a ┆ b ┆ c ┆ d ┆ new_a ┆ ratio │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │\n", + "╞══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════╡\n", + "│ 40554.594758 ┆ 40553.648222 ┆ 40555.409818 ┆ 40555.975997 ┆ 40559.594758 ┆ 0.999877 │\n", + "│ 40556.425904 ┆ 40553.38974 ┆ 40555.456314 ┆ 40554.628836 ┆ 40561.425904 ┆ 0.999877 │\n", + "│ 40554.444992 ┆ 40554.86456 ┆ 40554.680546 ┆ 40555.612513 ┆ 40559.444992 ┆ 0.999877 │\n", + "│ 40556.842544 ┆ 40553.290233 ┆ 40555.018377 ┆ 40555.304892 ┆ 40561.842544 ┆ 0.999877 │\n", + "│ 40555.926347 ┆ 40553.628159 ┆ 40555.399205 ┆ 40556.433585 ┆ 40560.926347 ┆ 0.999877 │\n", + "│ 40555.649944 ┆ 40554.988962 ┆ 40554.095679 ┆ 40554.4112 ┆ 40560.649944 ┆ 0.999877 │\n", + "└──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────┘" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "df[\"ratio\"] = df.eval(\"a / new_a\")\n", + "df = df.with_columns((pl.col(\"a\") / pl.col(\"new_a\")).alias(\"ratio\"))\n", "df" ] }, @@ -658,7 +989,51 @@ "id": "8b275b5b", "metadata": {}, "source": [ - "Evaluate can also be used to create new boolean columns using, for example, a string `\"a > 0.5\"` in the above example." + "You can also create boolean columns the same way:" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "f301c8cb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 7)
abcdnew_aratioa_gt_0.5
f64f64f64f64f64f64bool
40554.59475840553.64822240555.40981840555.97599740559.5947580.999877true
40556.42590440553.3897440555.45631440554.62883640561.4259040.999877true
40554.44499240554.8645640554.68054640555.61251340559.4449920.999877true
40556.84254440553.29023340555.01837740555.30489240561.8425440.999877true
40555.92634740553.62815940555.39920540556.43358540560.9263470.999877true
40555.64994440554.98896240554.09567940554.411240560.6499440.999877true
" + ], + "text/plain": [ + "shape: (6, 7)\n", + "┌──────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────┬──────────┐\n", + "│ a ┆ b ┆ c ┆ d ┆ new_a ┆ ratio ┆ a_gt_0.5 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ bool │\n", + "╞══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════╪══════════╡\n", + "│ 40554.594758 ┆ 40553.648222 ┆ 40555.409818 ┆ 40555.975997 ┆ 40559.594758 ┆ 0.999877 ┆ true │\n", + "│ 40556.425904 ┆ 40553.38974 ┆ 40555.456314 ┆ 40554.628836 ┆ 40561.425904 ┆ 0.999877 ┆ true │\n", + "│ 40554.444992 ┆ 40554.86456 ┆ 40554.680546 ┆ 40555.612513 ┆ 40559.444992 ┆ 0.999877 ┆ true │\n", + "│ 40556.842544 ┆ 40553.290233 ┆ 40555.018377 ┆ 40555.304892 ┆ 40561.842544 ┆ 0.999877 ┆ true │\n", + "│ 40555.926347 ┆ 40553.628159 ┆ 40555.399205 ┆ 40556.433585 ┆ 40560.926347 ┆ 0.999877 ┆ true │\n", + "│ 40555.649944 ┆ 40554.988962 ┆ 40554.095679 ┆ 40554.4112 ┆ 40560.649944 ┆ 0.999877 ┆ true │\n", + "└──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────┴──────────┘" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = df.with_columns((pl.col(\"a\") > 0.5).alias(\"a_gt_0.5\"))\n", + "df" ] } ], @@ -687,7 +1062,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/joins.ipynb b/joins.ipynb index e58d2d3..a1b57a3 100644 --- a/joins.ipynb +++ b/joins.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "51a55374", "metadata": { "tags": [ @@ -64,10 +64,98 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "f5ef4f37", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " county pop\n", + "0 Los Angeles 9878554\n", + "1 Orange 2997033\n", + "2 Ventura 798364\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
countypop
ca0Los Angeles9878554
1Orange2997033
2Ventura798364
il0Cook5285107
1DeKalb103729
2Will673586
\n", + "
" + ], + "text/plain": [ + " county pop\n", + "ca 0 Los Angeles 9878554\n", + " 1 Orange 2997033\n", + " 2 Ventura 798364\n", + "il 0 Cook 5285107\n", + " 1 DeKalb 103729\n", + " 2 Will 673586" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import pandas as pd\n", "\n", @@ -130,10 +218,93 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "53c66d5d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
key1key2ABCD
0K0K0A0B0C0D0
1K1K0A2B2C1D1
2K1K0A2B2C2D2
3K2K0NaNNaNC3D3
\n", + "
" + ], + "text/plain": [ + " key1 key2 A B C D\n", + "0 K0 K0 A0 B0 C0 D0\n", + "1 K1 K0 A2 B2 C1 D1\n", + "2 K1 K0 A2 B2 C2 D2\n", + "3 K2 K0 NaN NaN C3 D3" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "left = pd.DataFrame(\n", " {\n", @@ -167,10 +338,83 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "5e73608f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
key1key2ABCD
0K0K0A0B0C0D0
1K1K0A2B2C1D1
2K1K0A2B2C2D2
\n", + "
" + ], + "text/plain": [ + " key1 key2 A B C D\n", + "0 K0 K0 A0 B0 C0 D0\n", + "1 K1 K0 A2 B2 C1 D1\n", + "2 K1 K0 A2 B2 C2 D2" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pd.merge(left, right, on=[\"key1\", \"key2\"], how=\"inner\")" ] @@ -187,10 +431,120 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "5d209fb9", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
key1key2ABCD_merge
0K0K0A0B0C0D0both
1K0K1A1B1NaNNaNleft_only
2K1K0A2B2C1D1both
3K1K0A2B2C2D2both
4K2K0NaNNaNC3D3right_only
5K2K1A3B3NaNNaNleft_only
\n", + "
" + ], + "text/plain": [ + " key1 key2 A B C D _merge\n", + "0 K0 K0 A0 B0 C0 D0 both\n", + "1 K0 K1 A1 B1 NaN NaN left_only\n", + "2 K1 K0 A2 B2 C1 D1 both\n", + "3 K1 K0 A2 B2 C2 D2 both\n", + "4 K2 K0 NaN NaN C3 D3 right_only\n", + "5 K2 K1 A3 B3 NaN NaN left_only" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pd.merge(left, right, on=[\"key1\", \"key2\"], how=\"outer\", indicator=True)" ] @@ -255,7 +609,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/missing-values.ipynb b/missing-values.ipynb index 262f015..08aed7c 100644 --- a/missing-values.ipynb +++ b/missing-values.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "51a55374", "metadata": { "tags": [ @@ -61,10 +61,68 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "535ef959", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
numbers
05.0
127.3
2NaN
3-16.0
\n", + "
" + ], + "text/plain": [ + " numbers\n", + "0 5.0\n", + "1 27.3\n", + "2 NaN\n", + "3 -16.0" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import numpy as np\n", "import pandas as pd\n", @@ -83,10 +141,73 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "bf317bce", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
numbers
0<NA>
127.3
2NaN
3-16
4None
\n", + "
" + ], + "text/plain": [ + " numbers\n", + "0 \n", + "1 27.3\n", + "2 NaN\n", + "3 -16\n", + "4 None" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "numbers = pd.DataFrame([pd.NA, 27.3, np.nan, -16, None], columns=[\"numbers\"])\n", "numbers" @@ -102,10 +223,78 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "b1c864d9", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
fruit
0orange
1NaN
2apple
3None
4banana
5<NA>
\n", + "
" + ], + "text/plain": [ + " fruit\n", + "0 orange\n", + "1 NaN\n", + "2 apple\n", + "3 None\n", + "4 banana\n", + "5 " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "fruits = pd.DataFrame(\n", " [\"orange\", np.nan, \"apple\", None, \"banana\", pd.NA], columns=[\"fruit\"]\n", @@ -123,10 +312,78 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "cbcfe630", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
fruit
0False
1True
2False
3True
4False
5True
\n", + "
" + ], + "text/plain": [ + " fruit\n", + "0 False\n", + "1 True\n", + "2 False\n", + "3 True\n", + "4 False\n", + "5 True" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "fruits.isna()" ] @@ -151,10 +408,83 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "c96f89e6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ABCD
0NaN2.0NaN0
13.04.0NaN1
25.0NaNNaN<NA>
3NaN3.0NaN4
\n", + "
" + ], + "text/plain": [ + " A B C D\n", + "0 NaN 2.0 NaN 0\n", + "1 3.0 4.0 NaN 1\n", + "2 5.0 NaN NaN \n", + "3 NaN 3.0 NaN 4" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nan_df = pd.DataFrame(\n", " [\n", @@ -179,10 +509,91 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "a6bc5fe2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/4054961691.py:1: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " nan_df.fillna(0)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ABCD
00.02.00.00
13.04.00.01
25.00.00.00
30.03.00.04
\n", + "
" + ], + "text/plain": [ + " A B C D\n", + "0 0.0 2.0 0.0 0\n", + "1 3.0 4.0 0.0 1\n", + "2 5.0 0.0 0.0 0\n", + "3 0.0 3.0 0.0 4" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nan_df.fillna(0)" ] @@ -197,10 +608,91 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "e65b67c2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/2397886090.py:1: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " nan_df.fillna(value={\"A\": 0, \"B\": 1, \"C\": 2, \"D\": 3})\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ABCD
00.02.02.00
13.04.02.01
25.01.02.03
30.03.02.04
\n", + "
" + ], + "text/plain": [ + " A B C D\n", + "0 0.0 2.0 2.0 0\n", + "1 3.0 4.0 2.0 1\n", + "2 5.0 1.0 2.0 3\n", + "3 0.0 3.0 2.0 4" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nan_df.fillna(value={\"A\": 0, \"B\": 1, \"C\": 2, \"D\": 3})" ] @@ -215,20 +707,186 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "2a19e196", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/1353804149.py:1: FutureWarning: DataFrame.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.\n", + " nan_df.fillna(method=\"ffill\")\n", + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/1353804149.py:1: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " nan_df.fillna(method=\"ffill\")\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ABCD
0NaN2.0NaN0
13.04.0NaN1
25.04.0NaN1
35.03.0NaN4
\n", + "
" + ], + "text/plain": [ + " A B C D\n", + "0 NaN 2.0 NaN 0\n", + "1 3.0 4.0 NaN 1\n", + "2 5.0 4.0 NaN 1\n", + "3 5.0 3.0 NaN 4" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nan_df.fillna(method=\"ffill\")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "8b5b001e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/2505504399.py:1: FutureWarning: DataFrame.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.\n", + " nan_df.fillna(method=\"bfill\")\n", + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/2505504399.py:1: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " nan_df.fillna(method=\"bfill\")\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ABCD
03.02.0NaN0
13.04.0NaN1
25.03.0NaN4
3NaN3.0NaN4
\n", + "
" + ], + "text/plain": [ + " A B C D\n", + "0 3.0 2.0 NaN 0\n", + "1 3.0 4.0 NaN 1\n", + "2 5.0 3.0 NaN 4\n", + "3 NaN 3.0 NaN 4" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nan_df.fillna(method=\"bfill\")" ] @@ -251,10 +909,91 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "558e7a23", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/1730877720.py:1: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " nan_df.fillna(value={\"A\": 0, \"B\": 1, \"C\": 2, \"D\": 3}, limit=1)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ABCD
00.02.02.00
13.04.0NaN1
25.01.0NaN3
3NaN3.0NaN4
\n", + "
" + ], + "text/plain": [ + " A B C D\n", + "0 0.0 2.0 2.0 0\n", + "1 3.0 4.0 NaN 1\n", + "2 5.0 1.0 NaN 3\n", + "3 NaN 3.0 NaN 4" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nan_df.fillna(value={\"A\": 0, \"B\": 1, \"C\": 2, \"D\": 3}, limit=1)" ] @@ -269,20 +1008,84 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "0e3a81c1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "1 3.0\n", + "2 5.0\n", + "Name: A, dtype: float64" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nan_df[\"A\"].dropna(axis=0) # on a single column" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "9c1e312e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
0
1
2
3
\n", + "
" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: []\n", + "Index: [0, 1, 2, 3]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nan_df.dropna(axis=1)" ] @@ -297,10 +1100,83 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "3296ea35", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ABCD
0NaN2.0NaN0
13.04.0NaN1
25.0NaNNaN<NA>
3NaN3.0NaN4
\n", + "
" + ], + "text/plain": [ + " A B C D\n", + "0 NaN 2.0 NaN 0\n", + "1 3.0 4.0 NaN 1\n", + "2 5.0 NaN NaN \n", + "3 NaN 3.0 NaN 4" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nan_df.dropna(how=\"all\")" ] @@ -323,10 +1199,67 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "d3e1af81", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ABCD
13.04.0NaN1
25.0NaNNaN<NA>
\n", + "
" + ], + "text/plain": [ + " A B C D\n", + "1 3.0 4.0 NaN 1\n", + "2 5.0 NaN NaN " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "nan_df[nan_df[\"A\"].notna()]" ] @@ -345,10 +1278,71 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "45668d12", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ABC
0345
1-74-99
2-9965
\n", + "
" + ], + "text/plain": [ + " A B C\n", + "0 3 4 5\n", + "1 -7 4 -99\n", + "2 -99 6 5" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "stata_df = pd.DataFrame([[3, 4, 5], [-7, 4, -99], [-99, 6, 5]], columns=list(\"ABC\"))\n", "\n", @@ -365,10 +1359,71 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "c43070f4", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ABC
0345
1-74<NA>
2<NA>65
\n", + "
" + ], + "text/plain": [ + " A B C\n", + "0 3 4 5\n", + "1 -7 4 \n", + "2 6 5" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "stata_df.replace({-99: pd.NA})" ] @@ -383,10 +1438,71 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "12cc567a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ABC
0345
1<NA>4<NA>
2<NA>65
\n", + "
" + ], + "text/plain": [ + " A B C\n", + "0 3 4 5\n", + "1 4 \n", + "2 6 5" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "stata_df.replace({-99: pd.NA, -7: pd.NA})" ] @@ -413,10 +1529,99 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "057c5203", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
yearqtrprice
0202011.88
1202020.59
2202030.35
320204NaN
4202120.92
5202130.17
6202142.66
\n", + "
" + ], + "text/plain": [ + " year qtr price\n", + "0 2020 1 1.88\n", + "1 2020 2 0.59\n", + "2 2020 3 0.35\n", + "3 2020 4 NaN\n", + "4 2021 2 0.92\n", + "5 2021 3 0.17\n", + "6 2021 4 2.66" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "stocks = pd.DataFrame(\n", " {\n", @@ -462,10 +1667,75 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "87ab37d0", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
qtr1234
year
20201.880.590.35NaN
2021NaN0.920.172.66
\n", + "
" + ], + "text/plain": [ + "qtr 1 2 3 4\n", + "year \n", + "2020 1.88 0.59 0.35 NaN\n", + "2021 NaN 0.92 0.17 2.66" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "stocks.pivot(columns=\"qtr\", values=\"price\", index=\"year\")" ] @@ -492,7 +1762,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "92a914e3", "metadata": {}, "outputs": [], @@ -517,10 +1787,78 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "cc904247", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
namesmokerage
0Ikaiano34
1Olettano88
2Leriahpreviously75
3Dashayno47
\n", + "
" + ], + "text/plain": [ + " name smoker age\n", + "0 Ikaia no 34\n", + "1 Oletta no 88\n", + "2 Leriah previously 75\n", + "3 Dashay no 47" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "health_cut = health.iloc[:-1, :]\n", "health_cut" @@ -536,10 +1874,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "546bc395", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "smoker\n", + "no 3\n", + "previously 1\n", + "yes 0\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "health_cut[\"smoker\"].value_counts()" ] @@ -554,10 +1907,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "efb6c6ab", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/3998383890.py:1: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n", + " health_cut.groupby(\"smoker\")[\"age\"].mean()\n" + ] + }, + { + "data": { + "text/plain": [ + "smoker\n", + "no 56.333333\n", + "previously 75.000000\n", + "yes NaN\n", + "Name: age, dtype: float64" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "health_cut.groupby(\"smoker\")[\"age\"].mean()" ] @@ -596,7 +1972,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/numbers.ipynb b/numbers.ipynb index 388db20..129d361 100644 --- a/numbers.ipynb +++ b/numbers.ipynb @@ -20,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "51a55374", "metadata": { "tags": [ @@ -39,7 +39,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "6c89ca3d", "metadata": {}, "outputs": [], @@ -62,10 +62,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "18f1ee4c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "np.int64(336776)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights[\"dest\"].count()" ] @@ -80,10 +91,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "161a24ea", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "dest\n", + "ORD 17283\n", + "ATL 17215\n", + "LAX 16174\n", + "BOS 15508\n", + "MCO 14082\n", + " ... \n", + "MTJ 15\n", + "SBN 10\n", + "ANC 8\n", + "LEX 1\n", + "LGA 1\n", + "Name: count, Length: 105, dtype: int64" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights[\"dest\"].value_counts()" ] @@ -98,10 +132,124 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "8554277a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
mean_delaycount_flights
dest
ORD13.57048417283
ATL12.50982417215
LAX9.40134416174
BOS8.73061315508
MCO11.27599814082
.........
MTJ17.64285715
SBN21.10000010
ANC12.8750008
LGANaN1
LEX-9.0000001
\n", + "

105 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " mean_delay count_flights\n", + "dest \n", + "ORD 13.570484 17283\n", + "ATL 12.509824 17215\n", + "LAX 9.401344 16174\n", + "BOS 8.730613 15508\n", + "MCO 11.275998 14082\n", + "... ... ...\n", + "MTJ 17.642857 15\n", + "SBN 21.100000 10\n", + "ANC 12.875000 8\n", + "LGA NaN 1\n", + "LEX -9.000000 1\n", + "\n", + "[105 rows x 2 columns]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " flights.groupby([\"dest\"])\n", @@ -123,10 +271,111 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "061decae", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
miles
tailnum
D942DN3418
N0EGMQ250866
N10156115966
N102UW25722
N103US24619
......
N997DL54669
N998AT15432
N998DL66052
N999DN54623
N9EAMQ167317
\n", + "

4043 rows × 1 columns

\n", + "
" + ], + "text/plain": [ + " miles\n", + "tailnum \n", + "D942DN 3418\n", + "N0EGMQ 250866\n", + "N10156 115966\n", + "N102UW 25722\n", + "N103US 24619\n", + "... ...\n", + "N997DL 54669\n", + "N998AT 15432\n", + "N998DL 66052\n", + "N999DN 54623\n", + "N9EAMQ 167317\n", + "\n", + "[4043 rows x 1 columns]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(flights.groupby(\"tailnum\").agg(miles=(\"distance\", \"sum\")))" ] @@ -141,10 +390,111 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "ecdb5630", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
n_cancelled
dest
ABQ0
ACK0
ALB20
ANC0
ATL317
......
TPA59
TUL16
TVC5
TYS52
XNA25
\n", + "

105 rows × 1 columns

\n", + "
" + ], + "text/plain": [ + " n_cancelled\n", + "dest \n", + "ABQ 0\n", + "ACK 0\n", + "ALB 20\n", + "ANC 0\n", + "ATL 317\n", + "... ...\n", + "TPA 59\n", + "TUL 16\n", + "TVC 5\n", + "TYS 52\n", + "XNA 25\n", + "\n", + "[105 rows x 1 columns]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(flights.groupby(\"dest\").agg(n_cancelled=(\"dep_time\", lambda x: x.isnull().sum())))" ] @@ -188,10 +538,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "a473cd56", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "np.int64(4983)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights[\"distance\"].max()" ] @@ -206,10 +567,67 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "96285702", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
xy
013
152
27<NA>
\n", + "
" + ], + "text/plain": [ + " x y\n", + "0 1 3\n", + "1 5 2\n", + "2 7 " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = pd.DataFrame({\"x\": [1, 5, 7], \"y\": [3, 2, pd.NA]})\n", "df" @@ -225,10 +643,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "5bae5499", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 1\n", + "1 2\n", + "2 7\n", + "dtype: object" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.min(axis=1)" ] @@ -245,10 +677,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "3af6ce86", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n", + "divided by 3 gives\n", + "remainder:\n", + "[1, 2, 0, 1, 2, 0, 1, 2, 0, 1]\n", + "divisions:\n", + "[0, 0, 1, 1, 1, 2, 2, 2, 3, 3]\n" + ] + } + ], "source": [ "print([x for x in range(1, 11)])\n", "print(\"divided by 3 gives\")\n", @@ -268,10 +713,348 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "6be57e34", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
0201311517.05152.0830.081911.0UA1545N14228EWRIAH227.014005152013-01-01T10:00:00Z
1201311533.05294.0850.083020.0UA1714N24211LGAIAH227.014165292013-01-01T10:00:00Z
2201311542.05402.0923.085033.0AA1141N619AAJFKMIA160.010895402013-01-01T10:00:00Z
3201311544.0545-1.01004.01022-18.0B6725N804JBJFKBQN183.015765452013-01-01T10:00:00Z
4201311554.0600-6.0812.0837-25.0DL461N668DNLGAATL116.0762602013-01-01T11:00:00Z
............................................................
3367712013930NaN1455NaNNaN1634NaN9E3393NaNJFKDCANaN21314552013-09-30T18:00:00Z
3367722013930NaN2200NaNNaN2312NaN9E3525NaNLGASYRNaN1982202013-10-01T02:00:00Z
3367732013930NaN1210NaNNaN1330NaNMQ3461N535MQLGABNANaN76412102013-09-30T16:00:00Z
3367742013930NaN1159NaNNaN1344NaNMQ3572N511MQLGACLENaN41911592013-09-30T15:00:00Z
3367752013930NaN840NaNNaN1020NaNMQ3531N839MQLGARDUNaN4318402013-09-30T12:00:00Z
\n", + "

336776 rows × 19 columns

\n", + "
" + ], + "text/plain": [ + " year month day dep_time sched_dep_time dep_delay arr_time \\\n", + "0 2013 1 1 517.0 515 2.0 830.0 \n", + "1 2013 1 1 533.0 529 4.0 850.0 \n", + "2 2013 1 1 542.0 540 2.0 923.0 \n", + "3 2013 1 1 544.0 545 -1.0 1004.0 \n", + "4 2013 1 1 554.0 600 -6.0 812.0 \n", + "... ... ... ... ... ... ... ... \n", + "336771 2013 9 30 NaN 1455 NaN NaN \n", + "336772 2013 9 30 NaN 2200 NaN NaN \n", + "336773 2013 9 30 NaN 1210 NaN NaN \n", + "336774 2013 9 30 NaN 1159 NaN NaN \n", + "336775 2013 9 30 NaN 840 NaN NaN \n", + "\n", + " sched_arr_time arr_delay carrier flight tailnum origin dest \\\n", + "0 819 11.0 UA 1545 N14228 EWR IAH \n", + "1 830 20.0 UA 1714 N24211 LGA IAH \n", + "2 850 33.0 AA 1141 N619AA JFK MIA \n", + "3 1022 -18.0 B6 725 N804JB JFK BQN \n", + "4 837 -25.0 DL 461 N668DN LGA ATL \n", + "... ... ... ... ... ... ... ... \n", + "336771 1634 NaN 9E 3393 NaN JFK DCA \n", + "336772 2312 NaN 9E 3525 NaN LGA SYR \n", + "336773 1330 NaN MQ 3461 N535MQ LGA BNA \n", + "336774 1344 NaN MQ 3572 N511MQ LGA CLE \n", + "336775 1020 NaN MQ 3531 N839MQ LGA RDU \n", + "\n", + " air_time distance hour minute time_hour \n", + "0 227.0 1400 5 15 2013-01-01T10:00:00Z \n", + "1 227.0 1416 5 29 2013-01-01T10:00:00Z \n", + "2 160.0 1089 5 40 2013-01-01T10:00:00Z \n", + "3 183.0 1576 5 45 2013-01-01T10:00:00Z \n", + "4 116.0 762 6 0 2013-01-01T11:00:00Z \n", + "... ... ... ... ... ... \n", + "336771 NaN 213 14 55 2013-09-30T18:00:00Z \n", + "336772 NaN 198 22 0 2013-10-01T02:00:00Z \n", + "336773 NaN 764 12 10 2013-09-30T16:00:00Z \n", + "336774 NaN 419 11 59 2013-09-30T15:00:00Z \n", + "336775 NaN 431 8 40 2013-09-30T12:00:00Z \n", + "\n", + "[336776 rows x 19 columns]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "flights.assign(\n", " hour=lambda x: x[\"sched_dep_time\"] // 100,\n", @@ -291,10 +1074,79 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "a3ffc085", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
yearmoney
02001105.000000
12002110.250000
22003115.762500
32004121.550625
42005127.628156
\n", + "
" + ], + "text/plain": [ + " year money\n", + "0 2001 105.000000\n", + "1 2002 110.250000\n", + "2 2003 115.762500\n", + "3 2004 121.550625\n", + "4 2005 127.628156" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import numpy as np\n", "\n", @@ -316,10 +1168,821 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "aaad2abb", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:35:12.580055\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "money.plot(x=\"year\", y=\"money\");" ] @@ -334,10 +1997,767 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "bbcf4d7c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:35:12.673392\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "money.plot(x=\"year\", y=\"money\", logy=True);" ] @@ -366,10 +2786,79 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "ec1a09ca", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
yearmoney
02001105.00
12002110.25
22003115.76
32004121.55
42005127.63
\n", + "
" + ], + "text/plain": [ + " year money\n", + "0 2001 105.00\n", + "1 2002 110.25\n", + "2 2003 115.76\n", + "3 2004 121.55\n", + "4 2005 127.63" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "money.head().round(2)" ] @@ -384,10 +2873,79 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "9a306519", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
yearmoney
02001105.0
12002110.2
22003115.8
32004121.6
42005127.6
\n", + "
" + ], + "text/plain": [ + " year money\n", + "0 2001 105.0\n", + "1 2002 110.2\n", + "2 2003 115.8\n", + "3 2004 121.6\n", + "4 2005 127.6" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "money.head().round({\"year\": 0, \"money\": 1})" ] @@ -402,10 +2960,79 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "fb9dfb59", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
yearmoney
452046900.0
4620471000.0
4720481000.0
4820491100.0
4920501100.0
\n", + "
" + ], + "text/plain": [ + " year money\n", + "45 2046 900.0\n", + "46 2047 1000.0\n", + "47 2048 1000.0\n", + "48 2049 1100.0\n", + "49 2050 1100.0" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "money.tail().round({\"year\": 0, \"money\": -2})" ] @@ -420,10 +3047,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "f0290c0a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 100.0\n", + "1 110.0\n", + "2 120.0\n", + "3 120.0\n", + "4 130.0\n", + "Name: money, dtype: float64" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "money[\"money\"].head().apply(lambda x: float(f'{float(f\"{x:.2g}\"):g}'))" ] @@ -438,10 +3081,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "6a6a4644", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "array([2., 2., 1.])" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "np.round([1.5, 2.5, 1.4])" ] @@ -456,10 +3110,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "6270c3cc", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "array([27.15101204, 15.41426421, 76.7650241 , 70.51407739, 88.22482077,\n", + " 51.56875497, 63.03191713, 86.16710762, 72.35185576, 87.16963342])" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "real_nums = 100 * np.random.random(size=10)\n", "real_nums" @@ -467,20 +3133,42 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "dc3608d6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "array([28., 16., 77., 71., 89., 52., 64., 87., 73., 88.])" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "np.ceil(real_nums)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "d577bb21", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "array([27., 15., 76., 70., 88., 51., 63., 86., 72., 87.])" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "np.floor(real_nums)" ] @@ -495,10 +3183,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "dca417b7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 105.0\n", + "1 111.0\n", + "2 116.0\n", + "3 122.0\n", + "4 128.0\n", + "Name: money, dtype: float64" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "money[\"money\"].head().apply(np.ceil)" ] @@ -515,10 +3219,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "id": "c5d77818", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "45 943.425818\n", + "46 1934.022928\n", + "47 2974.149892\n", + "48 4066.283205\n", + "49 5213.023184\n", + "Name: money, dtype: float64" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "money[\"money\"].tail().cumsum()" ] @@ -547,20 +3267,134 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "id": "db1272a3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
xy
013
152
27<NA>
\n", + "
" + ], + "text/plain": [ + " x y\n", + "0 1 3\n", + "1 5 2\n", + "2 7 " + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "11ee0bac", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
xy
01.02.0
12.01.0
23.0NaN
\n", + "
" + ], + "text/plain": [ + " x y\n", + "0 1.0 2.0\n", + "1 2.0 1.0\n", + "2 3.0 NaN" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.rank()" ] @@ -575,10 +3409,67 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "edc7bd81", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
xy
00.3333331.0
10.6666670.5
21.000000NaN
\n", + "
" + ], + "text/plain": [ + " x y\n", + "0 0.333333 1.0\n", + "1 0.666667 0.5\n", + "2 1.000000 NaN" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.rank(pct=True)" ] @@ -597,10 +3488,1078 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "id": "9e499d1b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:35:12.825403\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "money[\"money_lag_5\"] = money[\"money\"].shift(5)\n", "money[\"money_lead_10\"] = money[\"money\"].shift(-10)\n", @@ -643,10 +4602,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "id": "81389031", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(190.92197566022773)" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "money[\"money\"].quantile(0.25)" ] @@ -661,10 +4631,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "id": "9eb35866", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0.00 105.000000\n", + "0.25 190.921976\n", + "0.50 347.101381\n", + "0.75 630.945970\n", + "Name: money, dtype: float64" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "money[\"money\"].quantile([0, 0.25, 0.5, 0.75])" ] @@ -684,10 +4669,71 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "id": "e2efca3a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
distance_sdcount
origindest
EWREGE1.0110
JFKEGE1.0103
\n", + "
" + ], + "text/plain": [ + " distance_sd count\n", + "origin dest \n", + "EWR EGE 1.0 110\n", + "JFK EGE 1.0 103" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " flights.groupby([\"origin\", \"dest\"])\n", @@ -713,20 +4759,3173 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "id": "b5ac4861", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:35:13.026671\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "flights[\"dep_delay\"].plot.hist(bins=50, title=\" Distribution: length of delay\");" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "id": "d4e76051", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2026-04-28T13:35:13.184191\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.10.0, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "flights.query(\"dep_delay <= 120\")[\"dep_delay\"].plot.hist(\n", " bins=50, title=\" Distribution: length of delay\"\n", @@ -792,7 +7991,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/prerequisites.ipynb b/prerequisites.ipynb index 68bea6d..07119a8 100644 --- a/prerequisites.ipynb +++ b/prerequisites.ipynb @@ -305,7 +305,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.0" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/rectangling.ipynb b/rectangling.ipynb index e565c1d..4c0ff4f 100644 --- a/rectangling.ipynb +++ b/rectangling.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "51a55374", "metadata": { "tags": [ @@ -56,10 +56,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "4d1c3fa4", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[10, 1.23, 'like this', True, None]\n" + ] + } + ], "source": [ "list_example = [10, 1.23, \"like this\", True, None]\n", "print(list_example)" @@ -75,10 +83,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "96c4b86b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[10, 1.23, 'like this', True, None, 'one more entry']\n" + ] + } + ], "source": [ "list_example.append(\"one more entry\")\n", "print(list_example)" @@ -94,10 +110,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "138ac0e0", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10\n", + "one more entry\n" + ] + } + ], "source": [ "print(list_example[0])\n", "print(list_example[-1])" @@ -123,10 +148,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "ef390263", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[10, 1.23, 'like this']\n", + "[True, None, 'one more entry']\n" + ] + } + ], "source": [ "print(list_example[:3])\n", "print(list_example[-3:])" @@ -142,10 +176,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "13584953", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2, 4, 6, 8]\n" + ] + } + ], "source": [ "list_of_numbers = list(range(1, 11))\n", "start = 1\n", @@ -164,10 +206,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "f2bc8926", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[10, 9, 8, 7, 6, 5, 4, 3, 2, 1]\n" + ] + } + ], "source": [ "print(list_of_numbers[::-1])" ] @@ -192,10 +242,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "703fb0b2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[3.1415,\n", + " 16,\n", + " ['five', 4, 3],\n", + " (91, 93, 90),\n", + " 'Hello World!',\n", + " True,\n", + " None,\n", + " {'key': 'value', 'key2': 'value2'}]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "wacky_list = [\n", " 3.1415,\n", @@ -222,10 +290,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "28fe13d2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15]]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "multilayer_list = [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15]]\n", "multilayer_list" @@ -241,10 +320,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "73a71803", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "[x for little_list in multilayer_list for x in little_list]" ] @@ -269,7 +359,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "007dadfb", "metadata": {}, "outputs": [], @@ -287,7 +377,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "aeb70f97", "metadata": {}, "outputs": [], @@ -307,10 +397,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "fa1d828c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (4, 3)
alphabetagamma
stri64list[str]
"["0,1,2"]"1["a", "b", "c"]
"foo"1null
"[]"1[]
"["3,4"]"1["d", "e"]
" + ], + "text/plain": [ + "shape: (4, 3)\n", + "┌───────────┬──────┬─────────────────┐\n", + "│ alpha ┆ beta ┆ gamma │\n", + "│ --- ┆ --- ┆ --- │\n", + "│ str ┆ i64 ┆ list[str] │\n", + "╞═══════════╪══════╪═════════════════╡\n", + "│ [\"0,1,2\"] ┆ 1 ┆ [\"a\", \"b\", \"c\"] │\n", + "│ foo ┆ 1 ┆ null │\n", + "│ [] ┆ 1 ┆ [] │\n", + "│ [\"3,4\"] ┆ 1 ┆ [\"d\", \"e\"] │\n", + "└───────────┴──────┴─────────────────┘" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = pl.DataFrame(\n", " {\n", @@ -333,10 +454,44 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "0854bcf3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (7, 3)
alphabetagamma
stri64str
"["0,1,2"]"1"a"
"["0,1,2"]"1"b"
"["0,1,2"]"1"c"
"foo"1null
"[]"1null
"["3,4"]"1"d"
"["3,4"]"1"e"
" + ], + "text/plain": [ + "shape: (7, 3)\n", + "┌───────────┬──────┬───────┐\n", + "│ alpha ┆ beta ┆ gamma │\n", + "│ --- ┆ --- ┆ --- │\n", + "│ str ┆ i64 ┆ str │\n", + "╞═══════════╪══════╪═══════╡\n", + "│ [\"0,1,2\"] ┆ 1 ┆ a │\n", + "│ [\"0,1,2\"] ┆ 1 ┆ b │\n", + "│ [\"0,1,2\"] ┆ 1 ┆ c │\n", + "│ foo ┆ 1 ┆ null │\n", + "│ [] ┆ 1 ┆ null │\n", + "│ [\"3,4\"] ┆ 1 ┆ d │\n", + "│ [\"3,4\"] ┆ 1 ┆ e │\n", + "└───────────┴──────┴───────┘" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df.explode(\"gamma\")" ] @@ -384,7 +539,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "285ade5e", "metadata": {}, "outputs": [], @@ -407,10 +562,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "75399bf0", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "dict" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "type(json_data)" ] @@ -427,10 +593,36 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "8cc4f408", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[{'date': '1971 FEB',\n", + " 'value': '3.8',\n", + " 'label': '1971 JAN-MAR',\n", + " 'year': '1971',\n", + " 'month': 'February',\n", + " 'quarter': '',\n", + " 'sourceDataset': 'LMS',\n", + " 'updateDate': '2015-10-13T23:00:00.000Z'},\n", + " {'date': '1971 MAR',\n", + " 'value': '3.9',\n", + " 'label': '1971 FEB-APR',\n", + " 'year': '1971',\n", + " 'month': 'March',\n", + " 'quarter': '',\n", + " 'sourceDataset': 'LMS',\n", + " 'updateDate': '2015-10-13T23:00:00.000Z'}]" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "json_data[\"months\"][:2]" ] @@ -447,7 +639,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "05069731", "metadata": {}, "outputs": [], @@ -473,10 +665,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "eca7982f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'food': 'doughnut',\n", + " 'good_with': ['coffee', 'tea'],\n", + " 'flavour': None,\n", + " 'toppings': [{'id': 0, 'type': 'glazed'}, {'id': 1, 'type': 'sugar'}]}" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import json\n", "\n", @@ -494,10 +700,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "ea6f887d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'{\"food\": \"doughnut\", \"good_with\": [\"coffee\", \"tea\"], \"flavour\": null, \"toppings\": [{\"id\": 0, \"type\": \"glazed\"}, {\"id\": 1, \"type\": \"sugar\"}]}'" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "json_stream = json.dumps(result)\n", "json_stream" @@ -546,7 +763,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "1dbfd7c3", "metadata": {}, "outputs": [], @@ -566,10 +783,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "b3ea0e0b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (3, 6)
idname.firstname.lastname.givenname.familyname
i64strstrstrstrstr
1"Coleen""Volk"nullnullnull
nullnullnull"Mark""Regner"null
2nullnullnullnull"Faye Raker"
" + ], + "text/plain": [ + "shape: (3, 6)\n", + "┌──────┬────────────┬───────────┬────────────┬─────────────┬────────────┐\n", + "│ id ┆ name.first ┆ name.last ┆ name.given ┆ name.family ┆ name │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ str ┆ str ┆ str ┆ str ┆ str │\n", + "╞══════╪════════════╪═══════════╪════════════╪═════════════╪════════════╡\n", + "│ 1 ┆ Coleen ┆ Volk ┆ null ┆ null ┆ null │\n", + "│ null ┆ null ┆ null ┆ Mark ┆ Regner ┆ null │\n", + "│ 2 ┆ null ┆ null ┆ null ┆ null ┆ Faye Raker │\n", + "└──────┴────────────┴───────────┴────────────┴─────────────┴────────────┘" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "data = [\n", " {\"id\": 1, \"name\": {\"first\": \"Coleen\", \"last\": \"Volk\"}},\n", @@ -589,10 +836,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "9349a3a7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (3, 2)
idname
i64str
1"{"first": "Coleen", "last": "V…
null"{"given": "Mark", "family": "R…
2"Faye Raker"
" + ], + "text/plain": [ + "shape: (3, 2)\n", + "┌──────┬─────────────────────────────────┐\n", + "│ id ┆ name │\n", + "│ --- ┆ --- │\n", + "│ i64 ┆ str │\n", + "╞══════╪═════════════════════════════════╡\n", + "│ 1 ┆ {\"first\": \"Coleen\", \"last\": \"V… │\n", + "│ null ┆ {\"given\": \"Mark\", \"family\": \"R… │\n", + "│ 2 ┆ Faye Raker │\n", + "└──────┴─────────────────────────────────┘" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pl.json_normalize(data, max_level=0)" ] @@ -628,7 +905,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/regex.ipynb b/regex.ipynb index a8055fc..d8cb9d0 100644 --- a/regex.ipynb +++ b/regex.ipynb @@ -23,9 +23,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "<>:4: SyntaxWarning: invalid escape sequence '\\w'\n", + "<>:4: SyntaxWarning: invalid escape sequence '\\w'\n", + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60848/970224743.py:4: SyntaxWarning: invalid escape sequence '\\w'\n", + " re.findall(\"string \\w+\\s\", text)\n" + ] + }, + { + "data": { + "text/plain": [ + "['string cleaning ', 'string editing ']" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import re\n", "\n", @@ -57,9 +78,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "<>:2: SyntaxWarning: invalid escape sequence '\\w'\n", + "<>:2: SyntaxWarning: invalid escape sequence '\\w'\n", + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60848/3231031485.py:2: SyntaxWarning: invalid escape sequence '\\w'\n", + " re.sub(\"string \\w+\\s\", new_text, text)\n" + ] + }, + { + "data": { + "text/plain": [ + "'It is true that new text here! is a topic in this chapter. new text here! is another.'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "new_text = \"new text here! \"\n", "re.sub(\"string \\w+\\s\", new_text, text)" @@ -141,9 +183,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The greedy match is stackoverflo\n", + "The lazy match is stacko\n" + ] + } + ], "source": [ "test_string = \"stackoverflow\"\n", "greedy_regex = \"s.*o\"\n", @@ -171,9 +222,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "<>:2: SyntaxWarning: invalid escape sequence '\\$'\n", + "<>:2: SyntaxWarning: invalid escape sequence '\\$'\n", + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60848/3729052804.py:2: SyntaxWarning: invalid escape sequence '\\$'\n", + " re.findall(\"\\$(\\d{2}.\\d{2})\", text)\n" + ] + }, + { + "data": { + "text/plain": [ + "['45.34', '50.00']" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "text = \"Product 1 was $45.34, while product 2 came in at $50.00 however it was assessed that the $4.66 difference did not make up for the higher quality of product 2.\"\n", "re.findall(\"\\$(\\d{2}.\\d{2})\", text)" @@ -192,9 +264,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['30500.00 to 35000']" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sal_r_per = r\"\\b([0-9]{1,6}(?:\\.)?(?:[0-9]{1,2})?(?:\\s?-\\s?|\\s?to\\s?)[0-9]{1,6}(?:\\.)?(?:[0-9]{1,2})?)(?:\\s?per)\\b\"\n", "text = \"This job pays gbp 30500.00 to 35000 per year. Apply at number 100 per the below address.\"\n", @@ -233,7 +316,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" } }, "nbformat": 4, diff --git a/spreadsheets.ipynb b/spreadsheets.ipynb index 1c1f378..4f5ebfd 100644 --- a/spreadsheets.ipynb +++ b/spreadsheets.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "51a55374", "metadata": { "tags": [ @@ -68,10 +68,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "bf00d4c6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 5)
Student IDFull Namefavourite.foodmealPlanAGE
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only"null
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""five"
6"Güvenç Attila""Ice cream""Lunch only""6"
" + ], + "text/plain": [ + "shape: (6, 5)\n", + "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", + "│ Student ID ┆ Full Name ┆ favourite.food ┆ mealPlan ┆ AGE │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", + "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", + "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", + "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", + "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", + "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ null │\n", + "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ five │\n", + "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", + "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import polars as pl\n", "\n", @@ -93,10 +126,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "ecce26ea", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 5)
student_idfull_namefavourite_foodmeal_planage
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only"null
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""five"
6"Güvenç Attila""Ice cream""Lunch only""6"
" + ], + "text/plain": [ + "shape: (6, 5)\n", + "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", + "│ student_id ┆ full_name ┆ favourite_food ┆ meal_plan ┆ age │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", + "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", + "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", + "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", + "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", + "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ null │\n", + "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ five │\n", + "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", + "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "students.columns = [\n", " \"student_id\",\n", @@ -118,10 +184,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "22e45cc2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (6, 5)
student_idfull_namefavourite_foodmeal_planage
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only"null
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""5"
6"Güvenç Attila""Ice cream""Lunch only""6"
" + ], + "text/plain": [ + "shape: (6, 5)\n", + "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", + "│ student_id ┆ full_name ┆ favourite_food ┆ meal_plan ┆ age │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", + "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", + "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", + "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", + "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", + "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ null │\n", + "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ 5 │\n", + "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", + "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "students = pl.read_excel(\"data/students.xlsx\")\n", "students.columns = [\n", @@ -145,10 +244,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "f67490d3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Schema([('student_id', Int64),\n", + " ('full_name', String),\n", + " ('favourite_food', String),\n", + " ('meal_plan', Categorical(ordering='physical')),\n", + " ('age', Int64)])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "students = students.with_columns(\n", " [\n", @@ -186,10 +300,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "e37f9e3d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (5, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64i64i64stri64
"Adelie""Torgersen"39.118.71813750"male"2007
"Adelie""Torgersen"39.517.41863800"female"2007
"Adelie""Torgersen"40.318.01953250"female"2007
"Adelie""Torgersen"nullnullnullnull"NA"2007
"Adelie""Torgersen"36.719.31933450"female"2007
" + ], + "text/plain": [ + "shape: (5, 8)\n", + "┌─────────┬───────────┬───────────────┬───────────────┬──────────────┬─────────────┬────────┬──────┐\n", + "│ species ┆ island ┆ bill_length_m ┆ bill_depth_mm ┆ flipper_leng ┆ body_mass_g ┆ sex ┆ year │\n", + "│ --- ┆ --- ┆ m ┆ --- ┆ th_mm ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ --- ┆ f64 ┆ --- ┆ i64 ┆ str ┆ i64 │\n", + "│ ┆ ┆ f64 ┆ ┆ i64 ┆ ┆ ┆ │\n", + "╞═════════╪═══════════╪═══════════════╪═══════════════╪══════════════╪═════════════╪════════╪══════╡\n", + "│ Adelie ┆ Torgersen ┆ 39.1 ┆ 18.7 ┆ 181 ┆ 3750 ┆ male ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 39.5 ┆ 17.4 ┆ 186 ┆ 3800 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 40.3 ┆ 18.0 ┆ 195 ┆ 3250 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ null ┆ null ┆ null ┆ null ┆ NA ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 36.7 ┆ 19.3 ┆ 193 ┆ 3450 ┆ female ┆ 2007 │\n", + "└─────────┴───────────┴───────────────┴───────────────┴──────────────┴─────────────┴────────┴──────┘" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pl.read_excel(\n", " \"data/penguins.xlsx\",\n", @@ -209,10 +356,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "0f3a3b78", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['Torgersen Island', 'Biscoe Island', 'Dream Island']\n" + ] + } + ], "source": [ "penguins_dict = pl.read_excel(\n", " \"data/penguins.xlsx\",\n", @@ -231,10 +386,50 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "15495426", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Biscoe Island\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "shape: (5, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64i64i64stri64
"Adelie""Biscoe"37.818.31743400"female"2007
"Adelie""Biscoe"37.718.71803600"male"2007
"Adelie""Biscoe"35.919.21893800"female"2007
"Adelie""Biscoe"38.218.11853950"male"2007
"Adelie""Biscoe"38.817.21803800"male"2007
" + ], + "text/plain": [ + "shape: (5, 8)\n", + "┌─────────┬────────┬────────────────┬───────────────┬────────────────┬─────────────┬────────┬──────┐\n", + "│ species ┆ island ┆ bill_length_mm ┆ bill_depth_mm ┆ flipper_length ┆ body_mass_g ┆ sex ┆ year │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ _mm ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ f64 ┆ f64 ┆ --- ┆ i64 ┆ str ┆ i64 │\n", + "│ ┆ ┆ ┆ ┆ i64 ┆ ┆ ┆ │\n", + "╞═════════╪════════╪════════════════╪═══════════════╪════════════════╪═════════════╪════════╪══════╡\n", + "│ Adelie ┆ Biscoe ┆ 37.8 ┆ 18.3 ┆ 174 ┆ 3400 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Biscoe ┆ 37.7 ┆ 18.7 ┆ 180 ┆ 3600 ┆ male ┆ 2007 │\n", + "│ Adelie ┆ Biscoe ┆ 35.9 ┆ 19.2 ┆ 189 ┆ 3800 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Biscoe ┆ 38.2 ┆ 18.1 ┆ 185 ┆ 3950 ┆ male ┆ 2007 │\n", + "│ Adelie ┆ Biscoe ┆ 38.8 ┆ 17.2 ┆ 180 ┆ 3800 ┆ male ┆ 2007 │\n", + "└─────────┴────────┴────────────────┴───────────────┴────────────────┴─────────────┴────────┴──────┘" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "print(list(penguins_dict.keys())[1])\n", "list(penguins_dict.values())[1].head()" @@ -250,10 +445,49 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "151ba846", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (344, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64i64i64stri64
"Adelie""Torgersen"39.118.71813750"male"2007
"Adelie""Torgersen"39.517.41863800"female"2007
"Adelie""Torgersen"40.318.01953250"female"2007
"Adelie""Torgersen"nullnullnullnull"NA"2007
"Adelie""Torgersen"36.719.31933450"female"2007
……………………
"Chinstrap""Dream"55.819.82074000"male"2009
"Chinstrap""Dream"43.518.12023400"female"2009
"Chinstrap""Dream"49.618.21933775"male"2009
"Chinstrap""Dream"50.819.02104100"male"2009
"Chinstrap""Dream"50.218.71983775"female"2009
" + ], + "text/plain": [ + "shape: (344, 8)\n", + "┌───────────┬───────────┬──────────────┬──────────────┬──────────────┬─────────────┬────────┬──────┐\n", + "│ species ┆ island ┆ bill_length_ ┆ bill_depth_m ┆ flipper_leng ┆ body_mass_g ┆ sex ┆ year │\n", + "│ --- ┆ --- ┆ mm ┆ m ┆ th_mm ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ --- ┆ --- ┆ --- ┆ i64 ┆ str ┆ i64 │\n", + "│ ┆ ┆ f64 ┆ f64 ┆ i64 ┆ ┆ ┆ │\n", + "╞═══════════╪═══════════╪══════════════╪══════════════╪══════════════╪═════════════╪════════╪══════╡\n", + "│ Adelie ┆ Torgersen ┆ 39.1 ┆ 18.7 ┆ 181 ┆ 3750 ┆ male ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 39.5 ┆ 17.4 ┆ 186 ┆ 3800 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 40.3 ┆ 18.0 ┆ 195 ┆ 3250 ┆ female ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ null ┆ null ┆ null ┆ null ┆ NA ┆ 2007 │\n", + "│ Adelie ┆ Torgersen ┆ 36.7 ┆ 19.3 ┆ 193 ┆ 3450 ┆ female ┆ 2007 │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ Chinstrap ┆ Dream ┆ 55.8 ┆ 19.8 ┆ 207 ┆ 4000 ┆ male ┆ 2009 │\n", + "│ Chinstrap ┆ Dream ┆ 43.5 ┆ 18.1 ┆ 202 ┆ 3400 ┆ female ┆ 2009 │\n", + "│ Chinstrap ┆ Dream ┆ 49.6 ┆ 18.2 ┆ 193 ┆ 3775 ┆ male ┆ 2009 │\n", + "│ Chinstrap ┆ Dream ┆ 50.8 ┆ 19.0 ┆ 210 ┆ 4100 ┆ male ┆ 2009 │\n", + "│ Chinstrap ┆ Dream ┆ 50.2 ┆ 18.7 ┆ 198 ┆ 3775 ┆ female ┆ 2009 │\n", + "└───────────┴───────────┴──────────────┴──────────────┴──────────────┴─────────────┴────────┴──────┘" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "penguins = pl.concat(penguins_dict.values())\n", "penguins" @@ -285,10 +519,49 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "29987b9d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (14, 6)
Lots of people__UNNAMED__1__UNNAMED__2__UNNAMED__3__UNNAMED__4__UNNAMED__5
strstrstrstrdatestr
"David Bowie""musician""69""true"1947-01-08"2016-01-10 00:00:00"
"Carrie Fisher""actor""60""true"1956-10-21"2016-12-27 00:00:00"
"Chuck Berry""musician""90""true"1926-10-18"2017-03-18 00:00:00"
"Bill Paxton""actor""61""true"1955-05-17"2017-02-25 00:00:00"
"Prince""musician""57""true"1958-06-07"2016-04-21 00:00:00"
………………
"George Michael""musician""53""false"1963-06-25"2016-12-25 00:00:00"
"Some"nullnullnullnullnull
null"also like to write stuff"nullnullnullnull
nullnull"at the""bottom,"nullnull
nullnullnullnullnull"too!"
" + ], + "text/plain": [ + "shape: (14, 6)\n", + "┌────────────────┬──────────────┬──────────────┬──────────────┬──────────────┬─────────────────────┐\n", + "│ Lots of people ┆ __UNNAMED__1 ┆ __UNNAMED__2 ┆ __UNNAMED__3 ┆ __UNNAMED__4 ┆ __UNNAMED__5 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ str ┆ str ┆ date ┆ str │\n", + "╞════════════════╪══════════════╪══════════════╪══════════════╪══════════════╪═════════════════════╡\n", + "│ David Bowie ┆ musician ┆ 69 ┆ true ┆ 1947-01-08 ┆ 2016-01-10 00:00:00 │\n", + "│ Carrie Fisher ┆ actor ┆ 60 ┆ true ┆ 1956-10-21 ┆ 2016-12-27 00:00:00 │\n", + "│ Chuck Berry ┆ musician ┆ 90 ┆ true ┆ 1926-10-18 ┆ 2017-03-18 00:00:00 │\n", + "│ Bill Paxton ┆ actor ┆ 61 ┆ true ┆ 1955-05-17 ┆ 2017-02-25 00:00:00 │\n", + "│ Prince ┆ musician ┆ 57 ┆ true ┆ 1958-06-07 ┆ 2016-04-21 00:00:00 │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ George Michael ┆ musician ┆ 53 ┆ false ┆ 1963-06-25 ┆ 2016-12-25 00:00:00 │\n", + "│ Some ┆ null ┆ null ┆ null ┆ null ┆ null │\n", + "│ null ┆ also like to ┆ null ┆ null ┆ null ┆ null │\n", + "│ ┆ write stuff ┆ ┆ ┆ ┆ │\n", + "│ null ┆ null ┆ at the ┆ bottom, ┆ null ┆ null │\n", + "│ null ┆ null ┆ null ┆ null ┆ null ┆ too! │\n", + "└────────────────┴──────────────┴──────────────┴──────────────┴──────────────┴─────────────────────┘" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pl.read_excel(\n", " \"data/deaths.xlsx\",\n", @@ -306,10 +579,47 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "9d7a3db3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (10, 6)
Lots of people__UNNAMED__1__UNNAMED__2__UNNAMED__3__UNNAMED__4__UNNAMED__5
strstri64booldatedate
"David Bowie""musician"69true1947-01-082016-01-10
"Carrie Fisher""actor"60true1956-10-212016-12-27
"Chuck Berry""musician"90true1926-10-182017-03-18
"Bill Paxton""actor"61true1955-05-172017-02-25
"Prince""musician"57true1958-06-072016-04-21
"Alan Rickman""actor"69false1946-02-212016-01-14
"Florence Henderson""actor"82true1934-02-142016-11-24
"Harper Lee""author"89false1926-04-282016-02-19
"Zsa Zsa Gábor""actor"99true1917-02-062016-12-18
"George Michael""musician"53false1963-06-252016-12-25
" + ], + "text/plain": [ + "shape: (10, 6)\n", + "┌────────────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────────┐\n", + "│ Lots of people ┆ __UNNAMED__1 ┆ __UNNAMED__2 ┆ __UNNAMED__3 ┆ __UNNAMED__4 ┆ __UNNAMED__5 │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ i64 ┆ bool ┆ date ┆ date │\n", + "╞════════════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════════╡\n", + "│ David Bowie ┆ musician ┆ 69 ┆ true ┆ 1947-01-08 ┆ 2016-01-10 │\n", + "│ Carrie Fisher ┆ actor ┆ 60 ┆ true ┆ 1956-10-21 ┆ 2016-12-27 │\n", + "│ Chuck Berry ┆ musician ┆ 90 ┆ true ┆ 1926-10-18 ┆ 2017-03-18 │\n", + "│ Bill Paxton ┆ actor ┆ 61 ┆ true ┆ 1955-05-17 ┆ 2017-02-25 │\n", + "│ Prince ┆ musician ┆ 57 ┆ true ┆ 1958-06-07 ┆ 2016-04-21 │\n", + "│ Alan Rickman ┆ actor ┆ 69 ┆ false ┆ 1946-02-21 ┆ 2016-01-14 │\n", + "│ Florence Henderson ┆ actor ┆ 82 ┆ true ┆ 1934-02-14 ┆ 2016-11-24 │\n", + "│ Harper Lee ┆ author ┆ 89 ┆ false ┆ 1926-04-28 ┆ 2016-02-19 │\n", + "│ Zsa Zsa Gábor ┆ actor ┆ 99 ┆ true ┆ 1917-02-06 ┆ 2016-12-18 │\n", + "│ George Michael ┆ musician ┆ 53 ┆ false ┆ 1963-06-25 ┆ 2016-12-25 │\n", + "└────────────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┘" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pl.read_excel(\n", " \"data/deaths.xlsx\",\n", @@ -356,10 +666,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "15963e18", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (3, 2)
itemquantity
cati64
"brownie"10
"cupcake"5
"cookie"8
" + ], + "text/plain": [ + "shape: (3, 2)\n", + "┌─────────┬──────────┐\n", + "│ item ┆ quantity │\n", + "│ --- ┆ --- │\n", + "│ cat ┆ i64 │\n", + "╞═════════╪══════════╡\n", + "│ brownie ┆ 10 │\n", + "│ cupcake ┆ 5 │\n", + "│ cookie ┆ 8 │\n", + "└─────────┴──────────┘" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "bake_sale = pl.DataFrame(\n", " {\n", @@ -380,10 +720,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "1fc17141", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "bake_sale.write_excel(\"data/bake_sale.xlsx\")" ] @@ -408,10 +759,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "27e128f9", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Schema([('item', String), ('quantity', Int64)])" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pl.read_excel(\"data/bake_sale.xlsx\").schema" ] @@ -449,7 +811,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/strings.ipynb b/strings.ipynb index e401876..10d93ae 100644 --- a/strings.ipynb +++ b/strings.ipynb @@ -28,7 +28,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "478a847a", "metadata": {}, "outputs": [], @@ -49,10 +49,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "d7f4ea2d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "str" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "type(string_one)" ] @@ -67,7 +78,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "01379fe7", "metadata": {}, "outputs": [], @@ -77,10 +88,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "d88f7928", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'ban'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "var[:3]" ] @@ -95,10 +117,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "e03d95d1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'aaa'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "var[1::2]" ] @@ -115,10 +148,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "83ab201b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "6" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "len(var)" ] @@ -133,10 +177,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "7801bd5d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "\"This is a string. If I want to include a 'quote' inside a string, I use double quotes on the outside.\"" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "string_one + \". \" + string_two + \".\"" ] @@ -151,10 +206,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "138cef18", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "\"This is a string. If I want to include a 'quote' inside a string, I use double quotes on the outside\"" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "\". \".join([string_one, string_two])" ] @@ -169,10 +235,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "e11896f8", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['INPUT TEXT', 'input text', 'Input Text']\n" + ] + } + ], "source": [ "var = \"input TEXT\"\n", "var_list = [var.upper(), var.lower(), var.title()]\n", @@ -201,10 +275,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "bf0aadec", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INPUT TEXT; and \n", + "input text; and \n", + "Input Text\n" + ] + } + ], "source": [ "print(*var_list, sep=\"; and \\n\")" ] @@ -219,10 +303,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "a96f048c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'A boolean is either True or False, there are only 2 options.'" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " \"A boolean is either \"\n", @@ -255,10 +350,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "9dddf0da", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "You scored 15.32399\n" + ] + } + ], "source": [ "variable = 15.32399\n", "print(f\"You scored {variable}\")" @@ -274,10 +377,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "795e7c07", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "You scored 234.8246695201\n" + ] + } + ], "source": [ "print(f\"You scored {variable**2}\")" ] @@ -300,10 +411,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "1f3d3806", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "You scored +15.32\n" + ] + } + ], "source": [ "print(f\"You scored {variable:+.2f}\")" ] @@ -352,10 +471,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "0ccd65aa", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import string\n", "\n", @@ -372,10 +502,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "16205c36", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "string.ascii_letters" ] @@ -390,10 +531,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "0c67f5cd", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'0123456789'" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "string.digits" ] @@ -416,10 +568,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "16e9904a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Here is a \n", + " new line\n", + "Here is an \\n escaped new line \n" + ] + } + ], "source": [ "print(\"Here is a \\n new line\")\n", "print(\"Here is an \\\\n escaped new line \")" @@ -452,10 +614,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "af423bd1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "a\tb\n", + "A\tB\n" + ] + } + ], "source": [ "print(\"a\\tb\\nA\\tB\")" ] @@ -472,10 +643,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "c2b9c689", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "a\\tb\\nA\\tB\n" + ] + } + ], "source": [ "print(r\"a\\tb\\nA\\tB\")" ] @@ -496,10 +675,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "229ada3a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'Value is subjective'" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "\"Value is objective\".replace(\"objective\", \"subjective\")" ] @@ -514,10 +704,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "79f754dc", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'Value is subjective'" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "text = \"Value is objective\"\n", "old_substr = \"objective\"\n", @@ -549,10 +750,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "99675fee", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'a': '', 'e': '', 'i': '', 'o': '', 'u': ''}" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "example_text = \"Much recent work has focused on the influence of social capital on innovative outcomes. Little research has been done on disadvantaged groups who were often restricted from participation in social networks that provide information necessary for invention and innovation. Unique new data on African American inventors and patentees between 1843 and 1930 permit an empirical investigation of the relation between social capital and economic outcomes. I find that African Americans used both traditional, i.e., occupation-based, and nontraditional, i.e., civic, networks to maximize inventive output and that laws constraining social-capital formation are most negatively correlated with economically important inventive activity.\"\n", "vowels = \"aeiou\"\n", @@ -570,10 +782,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "e48763cb", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'Mch rcnt wrk hs fcsd n th nflnc f scl cptl n nnvtv tcms. Lttl rsrch hs bn dn n dsdvntgd grps wh wr ftn rstrctd frm prtcptn n scl ntwrks tht prvd nfrmtn ncssry fr nvntn nd nnvtn. Unq nw dt n Afrcn Amrcn nvntrs nd ptnts btwn 1843 nd 1930 prmt n mprcl nvstgtn f th rltn btwn scl cptl nd cnmc tcms. I fnd tht Afrcn Amrcns sd bth trdtnl, .., ccptn-bsd, nd nntrdtnl, .., cvc, ntwrks t mxmz nvntv tpt nd tht lws cnstrnng scl-cptl frmtn r mst ngtvly crrltd wth cnmclly mprtnt nvntv ctvty.'" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "translator = example_text.maketrans(translation_dict)\n", "example_text.translate(translator)" @@ -599,10 +822,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "id": "ac758b38", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'Example string with excess punctuation'" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "\"\".join(\n", " [\n", @@ -625,10 +859,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "id": "122619bf", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'This is a sentence'" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "\"This is a sentence and we will split it at character 18\"[:18]" ] @@ -643,10 +888,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "9fc432ed", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['This is a sentence', ' And another sentence', ' And a third sentence']" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "\"This is a sentence. And another sentence. And a third sentence\".split(\".\")" ] @@ -661,10 +917,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "6904e486", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['This is a ', '. And another ', '. And a third ', '']" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "\"This is a sentence. And another sentence. And a third sentence\".split(\"sentence\")" ] @@ -689,10 +956,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "id": "22f94993", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The word \"coffee\" appears 2 times.\n" + ] + } + ], "source": [ "text = \"At six o'clock we were waiting for coffee, \\n waiting for coffee and the charitable crumb \\n that was going to be served from a certain balcony \\n --like kings of old, or like a miracle. \\n It was still dark. One foot of the sun \\n steadied itself on a long ripple in the river.\"\n", "word = \"coffee\"\n", @@ -709,10 +984,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "id": "a351a11b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "35" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "text.find(word)" ] @@ -727,10 +1013,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "id": "8e0a7020", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'coffee'" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "text[text.find(word) : text.find(word) + len(word)]" ] @@ -745,10 +1042,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "id": "e18f64a3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "57" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "text.rfind(word)" ] @@ -775,10 +1083,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "id": "bbc3eb7b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['Ada', 'Adam', 'Elinor', 'Grace', 'Jean']" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "[name.capitalize() for name in [\"ada\", \"adam\", \"elinor\", \"grace\", \"jean\"]]" ] @@ -793,10 +1112,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "id": "c8a7f68b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 ada lovelace\n", + "1 adam smith\n", + "2 elinor ostrom\n", + "3 grace hopper\n", + "4 jean bartik\n", + "dtype: string" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import pandas as pd\n", "\n", @@ -817,10 +1152,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "id": "7cf149b5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 Ada Lovelace\n", + "1 Adam Smith\n", + "2 Elinor Ostrom\n", + "3 Grace Hopper\n", + "4 Jean Bartik\n", + "dtype: string" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dfs.str.title()" ] @@ -835,10 +1186,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "id": "26dc9a7b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 Ada Lovelace\n", + "1 Adam Smith\n", + "2 Elinor Ostrom\n", + "3 Grace Hopper\n", + "4 Jean Bartik\n", + "Name: names, dtype: string" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = pd.DataFrame(dfs, columns=[\"names\"])\n", "df[\"names\"].str.title()" @@ -892,10 +1259,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "id": "d7a29663", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 [ada, lovelace]\n", + "1 [adam, smith]\n", + "2 [elinor, ostrom]\n", + "3 [grace, hopper]\n", + "4 [jean, bartik]\n", + "Name: names, dtype: object" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[\"names\"].str.split(\" \")" ] @@ -910,10 +1293,79 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "id": "85a5cd2c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
01
0adalovelace
1adamsmith
2elinorostrom
3gracehopper
4jeanbartik
\n", + "
" + ], + "text/plain": [ + " 0 1\n", + "0 ada lovelace\n", + "1 adam smith\n", + "2 elinor ostrom\n", + "3 grace hopper\n", + "4 jean bartik" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[\"names\"].str.split(\" \", n=2, expand=True)" ] @@ -932,10 +1384,36 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "id": "2e8781ba", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "<>:1: SyntaxWarning: invalid escape sequence '\\w'\n", + "<>:1: SyntaxWarning: invalid escape sequence '\\w'\n", + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60781/3354940721.py:1: SyntaxWarning: invalid escape sequence '\\w'\n", + " df[\"names\"].str.extract(\"(\\w+)\", expand=False)\n" + ] + }, + { + "data": { + "text/plain": [ + "0 ada\n", + "1 adam\n", + "2 elinor\n", + "3 grace\n", + "4 jean\n", + "Name: names, dtype: string" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[\"names\"].str.extract(\"(\\w+)\", expand=False)" ] @@ -967,10 +1445,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "id": "ba13d894", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 lovelace\n", + "1 smith\n", + "2 ostrom\n", + "3 hopper\n", + "4 bartik\n", + "Name: names, dtype: object" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[\"names\"].str.split().str.get(-1)" ] @@ -985,10 +1479,79 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "id": "056147d6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
namestags
0ada lovelaceuk; cs
1adam smithuk; econ
2elinor ostromusa; econ
3grace hopperusa; cs
4jean bartikusa; cs
\n", + "
" + ], + "text/plain": [ + " names tags\n", + "0 ada lovelace uk; cs\n", + "1 adam smith uk; econ\n", + "2 elinor ostrom usa; econ\n", + "3 grace hopper usa; cs\n", + "4 jean bartik usa; cs" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df = pd.DataFrame(\n", " {\n", @@ -1015,10 +1578,91 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "id": "a5cbc10f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cseconukusa
01010
10110
20101
31001
41001
\n", + "
" + ], + "text/plain": [ + " cs econ uk usa\n", + "0 1 0 1 0\n", + "1 0 1 1 0\n", + "2 0 1 0 1\n", + "3 1 0 0 1\n", + "4 1 0 0 1" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[\"tags\"].str.get_dummies(\";\")" ] @@ -1089,7 +1733,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/vis-layers.ipynb b/vis-layers.ipynb index ba22bb2..062b285 100644 --- a/vis-layers.ipynb +++ b/vis-layers.ipynb @@ -42,10 +42,51 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "a86fb211", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "import pandas as pd\n", "from lets_plot import *\n", @@ -67,10 +108,156 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "39a6d993", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
manufacturermodeldisplyearcyltransdrvctyhwyflclass
rownames
1audia41.819994auto(l5)f18.029.0pcompact
2audia41.819994manual(m5)f21.029.0pcompact
3audia42.020084manual(m6)f20.031.0pcompact
4audia42.020084auto(av)f21.030.0pcompact
5audia42.819996auto(l5)f16.026.0pcompact
\n", + "
" + ], + "text/plain": [ + " manufacturer model displ year cyl trans drv cty hwy fl \\\n", + "rownames \n", + "1 audi a4 1.8 1999 4 auto(l5) f 18.0 29.0 p \n", + "2 audi a4 1.8 1999 4 manual(m5) f 21.0 29.0 p \n", + "3 audi a4 2.0 2008 4 manual(m6) f 20.0 31.0 p \n", + "4 audi a4 2.0 2008 4 auto(av) f 21.0 30.0 p \n", + "5 audi a4 2.8 1999 6 auto(l5) f 16.0 26.0 p \n", + "\n", + " class \n", + "rownames \n", + "1 compact \n", + "2 compact \n", + "3 compact \n", + "4 compact \n", + "5 compact " + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "mpg = pd.read_csv(\n", " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/mpg.csv\", index_col=0\n", @@ -117,20 +304,262 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "fe77349a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\", color=\"class\")) + geom_point())" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "e77b5640", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\", shape=\"class\")) + geom_point())" ] @@ -145,20 +574,262 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "ef221330", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\", size=\"class\")) + geom_point())" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "d042255e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\", alpha=\"class\")) + geom_point())" ] @@ -181,10 +852,130 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "618edcb4", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point(color=\"blue\"))" ] @@ -250,20 +1041,263 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "277a4c0f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point(size=4))" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "07247ba9", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_smooth(method=\"loess\", size=2))" ] @@ -298,20 +1332,207 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "4b20c825", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\", line=\"drv\")) + geom_smooth(method=\"loess\"))" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "84df3e78", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\", linetype=\"drv\")) + geom_smooth(method=\"loess\"))" ] @@ -330,10 +1551,146 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "c9e8d92f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\", color=\"drv\"))\n", @@ -362,10 +1719,143 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "b3916558", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point(aes(color=\"class\")) + geom_smooth())" ] @@ -382,10 +1872,222 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "38870eb5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -442,14 +2144,149 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "ae75c5c1", "metadata": { "tags": [ "remove-cell" ] }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\", color=\"drv\"))\n", @@ -470,10 +2307,136 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "cb651300", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point() + facet_wrap(\"cyl\"))" ] @@ -488,10 +2451,138 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "61481052", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point() + facet_grid(\"drv\", \"cyl\"))" ] @@ -508,10 +2599,139 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "adcd9079", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -522,10 +2742,137 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "ceb2a354", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg) + geom_point(aes(x=\"displ\", y=\"hwy\")) + facet_wrap(\"class\", nrow=2))" ] @@ -615,10 +2962,141 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "f379e31b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
caratcutcolorclaritydepthtablepricexyz
rownames
10.23IdealESI261.555.03263.953.982.43
20.21PremiumESI159.861.03263.893.842.31
30.23GoodEVS156.965.03274.054.072.31
40.29PremiumIVS262.458.03344.204.232.63
50.31GoodJSI263.358.03354.344.352.75
\n", + "
" + ], + "text/plain": [ + " carat cut color clarity depth table price x y z\n", + "rownames \n", + "1 0.23 Ideal E SI2 61.5 55.0 326 3.95 3.98 2.43\n", + "2 0.21 Premium E SI1 59.8 61.0 326 3.89 3.84 2.31\n", + "3 0.23 Good E VS1 56.9 65.0 327 4.05 4.07 2.31\n", + "4 0.29 Premium I VS2 62.4 58.0 334 4.20 4.23 2.63\n", + "5 0.31 Good J SI2 63.3 58.0 335 4.34 4.35 2.75" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "diamonds = pd.read_csv(\n", " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/diamonds.csv\",\n", @@ -633,10 +3111,130 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "d8faf1ab", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(diamonds, aes(x=\"cut\")) + geom_bar())" ] @@ -679,10 +3277,108 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "ca772dd5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(\n", " ggplot(\n", @@ -706,20 +3402,258 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "f8da7d91", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"drv\", color=\"drv\")) + geom_bar())" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "088e7550", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"drv\", fill=\"drv\")) + geom_bar())" ] @@ -735,10 +3669,130 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "id": "181c70d2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"drv\", fill=\"class\")) + geom_bar())" ] @@ -758,10 +3812,132 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "id": "a8e9c378", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"drv\", fill=\"class\")) + geom_bar(alpha=0.5, position=\"identity\"))" ] @@ -779,10 +3955,131 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "14205000", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"drv\", fill=\"class\")) + geom_bar(position=\"fill\"))" ] @@ -798,10 +4095,131 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "c33c4a03", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"drv\", fill=\"class\")) + geom_bar(position=\"dodge\"))" ] @@ -818,10 +4236,129 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "id": "ba4161de", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point())" ] @@ -843,10 +4380,130 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "id": "414ce7af", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point(position=\"jitter\"))" ] @@ -878,10 +4535,129 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "id": "9bc38aef", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(ggplot(mpg, aes(x=\"cty\", y=\"hwy\")) + geom_point())" ] @@ -995,7 +4771,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/webscraping-and-apis.ipynb b/webscraping-and-apis.ipynb index 5171a4d..4bd60c2 100644 --- a/webscraping-and-apis.ipynb +++ b/webscraping-and-apis.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "51a55374", "metadata": { "tags": [ @@ -74,7 +74,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "a7f62293", "metadata": {}, "outputs": [], @@ -101,10 +101,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "06108a4d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (10, 14)
rownamesnameyearmonthdayhourlatlongstatuscategorywindpressuretropicalstorm_force_diameterhurricane_force_diameter
i64stri64i64i64i64f64f64stri64i64i64strstr
1"Amy"1975627027.5-79.0"tropical depression"null251013nullnull
2"Amy"1975627628.5-79.0"tropical depression"null251013nullnull
3"Amy"19756271229.5-79.0"tropical depression"null251013nullnull
4"Amy"19756271830.5-79.0"tropical depression"null251013nullnull
5"Amy"1975628031.5-78.8"tropical depression"null251012nullnull
6"Amy"1975628632.4-78.7"tropical depression"null251012nullnull
7"Amy"19756281233.3-78.0"tropical depression"null251011nullnull
8"Amy"19756281834.0-77.0"tropical depression"null301006nullnull
9"Amy"1975629034.4-75.8"tropical storm"null351004nullnull
10"Amy"1975629634.0-74.8"tropical storm"null401002nullnull
" + ], + "text/plain": [ + "shape: (10, 14)\n", + "┌──────────┬──────┬──────┬───────┬───┬──────┬──────────┬─────────────────────┬─────────────────────┐\n", + "│ rownames ┆ name ┆ year ┆ month ┆ … ┆ wind ┆ pressure ┆ tropicalstorm_force ┆ hurricane_force_dia │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ _diameter ┆ meter │\n", + "│ i64 ┆ str ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ --- ┆ --- │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ str ┆ str │\n", + "╞══════════╪══════╪══════╪═══════╪═══╪══════╪══════════╪═════════════════════╪═════════════════════╡\n", + "│ 1 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 25 ┆ 1013 ┆ null ┆ null │\n", + "│ 2 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 25 ┆ 1013 ┆ null ┆ null │\n", + "│ 3 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 25 ┆ 1013 ┆ null ┆ null │\n", + "│ 4 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 25 ┆ 1013 ┆ null ┆ null │\n", + "│ 5 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 25 ┆ 1012 ┆ null ┆ null │\n", + "│ 6 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 25 ┆ 1012 ┆ null ┆ null │\n", + "│ 7 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 25 ┆ 1011 ┆ null ┆ null │\n", + "│ 8 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 30 ┆ 1006 ┆ null ┆ null │\n", + "│ 9 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 35 ┆ 1004 ┆ null ┆ null │\n", + "│ 10 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 40 ┆ 1002 ┆ null ┆ null │\n", + "└──────────┴──────┴──────┴───────┴───┴──────┴──────────┴─────────────────────┴─────────────────────┘" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pl.read_csv(\n", " \"https://vincentarelbundock.github.io/Rdatasets/csv/dplyr/storms.csv\", n_rows=10\n", @@ -136,10 +174,113 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "6107093c", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "url = \"https://api.beta.ons.gov.uk/v1/data?uri=/employmentandlabourmarket/peopleinwork/employmentandemployeetypes/timeseries/jp9z/lms/previous/v108\"\n", "\n", @@ -202,12 +343,60 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "380ca743", "metadata": { "lines_to_next_cell": 2 }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61971/1219974464.py:18: MapWithoutReturnDtypeWarning: Calling `map_elements` without specifying `return_dtype` can lead to unpredictable results. Specify `return_dtype` to silence this warning.\n", + " .with_columns(pl.col(\"country\").map_elements(lambda x: textwrap.fill(x, 10)))\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "shape: (5, 3)
economycountryEN.GHG.ALL.PC.CE.AR5
strstrf64
"USA""United\n", + "States"18.921098
"CHN""China"9.982534
"ECS""Europe &\n", + "Central\n", + "Asia"9.171446
"EAS""East Asia\n", + "& Pacific"8.435052
"IND""India"2.621464
" + ], + "text/plain": [ + "shape: (5, 3)\n", + "┌─────────┬───────────┬──────────────────────┐\n", + "│ economy ┆ country ┆ EN.GHG.ALL.PC.CE.AR5 │\n", + "│ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ f64 │\n", + "╞═════════╪═══════════╪══════════════════════╡\n", + "│ USA ┆ United ┆ 18.921098 │\n", + "│ ┆ States ┆ │\n", + "│ CHN ┆ China ┆ 9.982534 │\n", + "│ ECS ┆ Europe & ┆ 9.171446 │\n", + "│ ┆ Central ┆ │\n", + "│ ┆ Asia ┆ │\n", + "│ EAS ┆ East Asia ┆ 8.435052 │\n", + "│ ┆ & Pacific ┆ │\n", + "│ IND ┆ India ┆ 2.621464 │\n", + "└─────────┴───────────┴──────────────────────┘" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# World Bank CO2 equivalent emissions (metric tons per capita)\n", "# https://data.worldbank.org/indicator/EN.GHG.ALL.PC.CE.AR5\n", @@ -235,10 +424,170 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "00379b93", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "lp.LetsPlot.setup_html()\n", "\n", @@ -386,10 +735,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "073d89e3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n\\n\\n\\n\\n\\n\\n\\n\\n'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "url = \"http://aeturrell.com/research\"\n", "page = requests.get(url)\n", @@ -409,10 +769,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "22f96be0", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ">\n", + "
\n", + "
\n", + " \n", + " data science\n", + " \n", + "
\n", + "
\n", + " \n", + " public policy\n", + " \n", + "
\n", + "
\n", + " \n", + " complexity science\n", + " \n", + "
\n", + " <\n" + ] + } + ], "source": [ "soup = BeautifulSoup(page.text, \"html.parser\")\n", "print(soup.prettify()[60000:60500])" @@ -429,10 +814,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "d82775de", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "

Blundell, Jack, Emma Duchini, Stefania Simion, and Arthur Turrell. \"Pay transparency and gender equality.\" American Economic Journal: Economic Policy (2024). doi: 10.1257/pol.20220766

" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Get all paragraphs\n", "all_paras = soup.find_all(\"p\")\n", @@ -451,10 +847,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "11321154", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'Blundell, Jack, Emma Duchini, Stefania Simion, and Arthur Turrell. \"Pay transparency and gender equality.\" American Economic Journal: Economic Policy (2024). doi: 10.1257/pol.20220766'" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "all_paras[1].text" ] @@ -470,10 +877,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "feac1fdd", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['Blundell, Jack, Emma Duchini, Stefania Simion, and Arthur Turrell. \"Pay transparency and gender equality.\" American Economic Journal: Economic Policy (2024). doi: 10.1257/pol.20220766',\n", + " 'Botta, Federico, Robin Lovelace, Laura Gilbert, and Arthur Turrell. \"Packaging code and data for reproducible research: A case study of journey time statistics.\" Environment and Planning B: Urban Analytics and City Science (2024): 23998083241267331. doi: 10.1177/23998083241267331',\n", + " 'Kalamara, Eleni, Arthur Turrell, Chris Redl, George Kapetanios, and Sujit Kapadia. \"Making text count: economic forecasting using newspaper text.\" Journal of Applied Econometrics 37, no. 5 (2022): 896-919. doi: 10.1002/jae.2907',\n", + " 'Turrell, A., Speigner, B., Copple, D., Djumalieva, J. and Thurgood, J., 2021. Is the UK’s productivity puzzle mostly driven by occupational mismatch? An analysis using big data on job vacancies. Labour Economics, 71, p.102013. doi: 10.1016/j.labeco.2021.102013',\n", + " 'Haldane, Andrew G., and Arthur E. Turrell. \"Drawing on different disciplines: macroeconomic agent-based models.\" Journal of Evolutionary Economics 29 (2019): 39-66. doi: 10.1007/s00191-018-0557-5',\n", + " 'Haldane, Andrew G., and Arthur E. Turrell. \"An interdisciplinary model for macroeconomics.\" Oxford Review of Economic Policy 34, no. 1-2 (2018): 219-251. doi: 10.1093/oxrep/grx051',\n", + " 'Braun-Munzinger, Karen, Z. Liu, and A. E. Turrell. \"An agent-based model of corporate bond trading.\" Quantitative Finance 18, no. 4 (2018): 591-608. doi: 10.1080/14697688.2017.1380310',\n", + " 'Turrell, A. E., M. Sherlock, and S. J. Rose. \"Efficient evaluation of collisional energy transfer terms for plasma particle simulations.\" Journal of Plasma Physics 82, no. 1 (2016): 905820107. doi: 10.1017/S0022377816000131',\n", + " 'Turrell, A. E., M. Sherlock, and S. J. Rose. \"Ultrafast collisional ion heating by electrostatic shocks.\" Nature Communications 6, no. 1 (2015): 8905. doi: 10.1038/ncomms9905',\n", + " 'Turrell, Arthur E., Mark Sherlock, and Steven J. Rose. \"Self-consistent inclusion of classical large-angle Coulomb collisions in plasma Monte Carlo simulations.\" Journal of Computational Physics 299 (2015): 144-155. doi: 10.1016/j.jcp.2015.06.034',\n", + " 'Turrell, Arthur E., Mark Sherlock, and Steven J. Rose. \"A Monte Carlo algorithm for degenerate plasmas.\" Journal of Computational Physics 249 (2013): 13-21. doi: 10.1016/j.jcp.2013.03.052',\n", + " 'Turrell, Arthur. \"Cutting through Complexity: How Data Science Can Help Policymakers Understand the World.\" In The Economy as a Complex Evolving System, Part IV. Sante Fe Institute, 2025. doi: https://doi.org/10.37911/9781947864665.11',\n", + " 'Duchini, Emma, Stefania Simion, and Arthur Turrell. \"A Review of the Effects of Pay Transparency.\" In Oxford Research Encyclopedia of Economics and Finance, Oxford University Press, 2024. doi: 10.1093/acrefore/9780190625979.013.860',\n", + " 'Turrell, Arthur, Bradley Speigner, Jyldyz Djumalieva, David Copple, and James Thurgood. \"6. Transforming Naturally Occurring Text Data into Economic Statistics.\" In Big Data for Twenty-First-Century Economic Statistics, pp. 173-208. University of Chicago Press, 2022. doi: 10.7208/chicago/9780226801391-008',\n", + " 'Turrell, Arthur. \"Agent-based models: understanding the economy from the bottom up\" In Quarterly Bulletin, Q4. Bank of England, 2016.',\n", + " 'Cohen, Samuel N., Giulia Mantoan, Lars Nesheim, Áureo de Paula, Arthur Turrell, and Lingyi Yang. Nowcasting using regression on signatures arXiv preprint arXiv:2305.10256v2 (2025).',\n", + " 'Van Dijcke, David, Marcus Buckmann, Arthur Turrell, and Tomas Key. \"Vacancy Posting, Firm Balance Sheets, and Pandemic Policy Interventions.\" Bank of England Staff Working Paper Series 1033 (2022).',\n", + " 'Draca, Mirko, Emma Duchini, Roland Rathelot, Arthur Turrell, and Giulia Vattuone. Revolution in Progress? The Rise of Remote Work in the UK. University of Warwick, Department of Economics, 2022.',\n", + " 'Hill, Edward, Marco Bardoscia, and Arthur Turrell. \"Solving heterogeneous general equilibrium economic models with deep reinforcement learning.\" arXiv arXiv:2103.16977 (2021).',\n", + " 'Turrell, Arthur, James Thurgood, David Copple, Jyldyz Djumalieva, and Bradley Speigner. \"Using online job vacancies to understand the UK labour market from the bottom-up.\" Bank of England Staff Working Papers 742 (2018).']" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "projects = soup.find_all(\"div\", class_=\"project-content listing-pub-info\")\n", "projects = [x.text.strip() for x in projects]\n", @@ -518,10 +955,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "0ada9ce7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "shape: (3, 4)\n", + "┌─────┬────────────┬───────────┬──────────┐\n", + "│ # ┆ First Name ┆ Last Name ┆ Username │\n", + "│ --- ┆ --- ┆ --- ┆ --- │\n", + "│ i64 ┆ str ┆ str ┆ str │\n", + "╞═════╪════════════╪═══════════╪══════════╡\n", + "│ 1 ┆ Mark ┆ Otto ┆ @mdo │\n", + "│ 2 ┆ Jacob ┆ Thornton ┆ @fat │\n", + "│ 3 ┆ Larry ┆ the Bird ┆ @twitter │\n", + "└─────┴────────────┴───────────┴──────────┘\n" + ] + } + ], "source": [ "import polars as pl\n", "\n", @@ -566,7 +1020,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/whole-game.ipynb b/whole-game.ipynb index eee6c2c..5498d0e 100644 --- a/whole-game.ipynb +++ b/whole-game.ipynb @@ -13,7 +13,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "51a55374", "metadata": { "tags": [ @@ -29,14 +29,116 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "209ef434", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "cluster_0\n", + "\n", + "Understand\n", + "\n", + "\n", + "\n", + "Import\n", + "\n", + "Import\n", + "\n", + "\n", + "\n", + "Clean\n", + "\n", + "Clean\n", + "\n", + "\n", + "\n", + "Import->Clean\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Analyse\n", + "\n", + "Analyse\n", + "\n", + "\n", + "\n", + "Clean->Analyse\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Visualise\n", + "\n", + "Visualise\n", + "\n", + "\n", + "\n", + "Visualise->Analyse\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Transform\n", + "\n", + "Transform\n", + "\n", + "\n", + "\n", + "Analyse->Transform\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Communicate\n", + "\n", + "Communicate\n", + "\n", + "\n", + "\n", + "Analyse->Communicate\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Transform->Visualise\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# | echo: false\n", "import graphviz\n", @@ -107,7 +209,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/workflow-basics.ipynb b/workflow-basics.ipynb index a0a7e47..27b2168 100644 --- a/workflow-basics.ipynb +++ b/workflow-basics.ipynb @@ -26,10 +26,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "23465996", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.15\n", + "44.666666666666664\n" + ] + } + ], "source": [ "print(1 / 200 * 30)\n", "print((59 + 73 + 2) / 3)" @@ -46,10 +55,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "bdd3c2c0", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.0\n" + ] + } + ], "source": [ "import numpy as np\n", "\n", @@ -67,10 +84,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "cead1264", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "12\n" + ] + } + ], "source": [ "x = 3 * 4\n", "print(x)" @@ -87,10 +112,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "91a44d83", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[1, 2, 3, 5, 7, 11, 13]\n" + ] + } + ], "source": [ "primes = [1, 2, 3, 5, 7, 11, 13]\n", "print(primes)" @@ -107,10 +140,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "1a526124", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[3, 6, 9, 15, 21, 33, 39]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "[element * 3 for element in primes]" ] @@ -154,10 +198,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "c061f9cc", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[2, 4, 6, 10, 14, 22, 26]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# define primes\n", "primes = [1, 2, 3, 5, 7, 11, 13]\n", @@ -193,10 +248,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "e77540e0", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[1, 2, 3, 5, 7, 11, 13]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "primes" ] @@ -212,10 +278,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "9dd0fd49", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "list" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "type(primes)" ] @@ -273,10 +350,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "a5998cb5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[1, 2, 3, 5, 7, 11, 13]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "primes" ] @@ -291,7 +379,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "ae55d78e", "metadata": {}, "outputs": [], @@ -313,7 +401,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "038c7d52", "metadata": {}, "outputs": [], @@ -369,10 +457,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "97af119d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "42" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sum(primes)" ] @@ -388,10 +487,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "14ce9b99", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "52" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sum(primes, start=10)" ] @@ -407,10 +517,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "f5e45616", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on built-in function sum in module builtins:\n", + "\n", + "sum(iterable, /, start=0)\n", + " Return the sum of a 'start' value (default: 0) plus an iterable of numbers\n", + "\n", + " When the iterable is empty, return the start value.\n", + " This function is intended specifically for use with numeric values and may\n", + " reject non-numeric types.\n", + "\n" + ] + } + ], "source": [ "help(sum)" ] @@ -528,7 +654,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/workflow-style.ipynb b/workflow-style.ipynb index 6921e86..7a1a2a4 100644 --- a/workflow-style.ipynb +++ b/workflow-style.ipynb @@ -148,10 +148,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "f0f5bb37", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (3, 4)
col3col4col1col2
strstru32f64
"a""alpha"10.0
"b""gamma"20.0
"a""gamma"10.0
" + ], + "text/plain": [ + "shape: (3, 4)\n", + "┌──────┬───────┬──────┬──────┐\n", + "│ col3 ┆ col4 ┆ col1 ┆ col2 │\n", + "│ --- ┆ --- ┆ --- ┆ --- │\n", + "│ str ┆ str ┆ u32 ┆ f64 │\n", + "╞══════╪═══════╪══════╪══════╡\n", + "│ a ┆ alpha ┆ 1 ┆ 0.0 │\n", + "│ b ┆ gamma ┆ 2 ┆ 0.0 │\n", + "│ a ┆ gamma ┆ 1 ┆ 0.0 │\n", + "└──────┴───────┴──────┴──────┘" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import polars as pl\n", "\n", @@ -271,7 +301,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/workflow-writing-code.ipynb b/workflow-writing-code.ipynb index 28e2edd..bb442d2 100644 --- a/workflow-writing-code.ipynb +++ b/workflow-writing-code.ipynb @@ -150,7 +150,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.0" + "version": "3.12.13" } }, "nbformat": 4, From 2ae8c35a23c9920aef5b16c0c787e090a816ccef Mon Sep 17 00:00:00 2001 From: Nwabueze Ugoh Date: Tue, 28 Apr 2026 13:49:11 +0100 Subject: [PATCH 31/34] Enhance command line documentation and update notebook outputs - Updated the command line chapter in `command-line.md` to improve formatting by replacing asterisks with underscores for emphasis. - Modified output displays in `functions.ipynb` and `iteration.ipynb` to reflect new data and execution counts, ensuring consistency and clarity in results. - Adjusted various execution counts across notebooks to maintain accurate tracking of code execution. --- command-line.md | 104 +++++++++++++++++++++++------------------------ functions.ipynb | 22 +++++----- iteration.ipynb | 106 ++++++++++++++++++++++++------------------------ 3 files changed, 116 insertions(+), 116 deletions(-) diff --git a/command-line.md b/command-line.md index 1079842..2cdce9b 100644 --- a/command-line.md +++ b/command-line.md @@ -1,6 +1,6 @@ # The Command Line {#sec-command-line} -In this chapter, you'll meet the *command line* and learn how to use it. Beyond a few key commands like `uv add ` you don't strictly need to know how to use the command line to follow the rest of this book. However, even a tiny bit of knowledge of the command line goes a long way in coding and will serve you well. +In this chapter, you'll meet the _command line_ and learn how to use it. Beyond a few key commands like `uv add ` you don't strictly need to know how to use the command line to follow the rest of this book. However, even a tiny bit of knowledge of the command line goes a long way in coding and will serve you well. To try out any of the commands in this chapter on your machine, you can select 'New Terminal' from the menu bar in Visual Studio Code (Mac and Linux), use the Windows Subsystem for Linux or git bash (Windows), or use a free [online terminal](https://cocalc.com/doc/terminal.html). @@ -8,17 +8,17 @@ This chapter has benefited from numerous sources, including absolutely excellent ## What is the command line? -The command line is a way to directly issue text-based commands to a computer one line at a time (as distinct from a graphical user interface, or GUI, that you navigate with a mouse). It goes under many names: shell, bash, terminal, CLI, and command line. These are actually different things but most people tend to use them to mean the same thing most of the time. The *shell* is the part of an operating system that you interact with but mostly people use shell to mean the command line. *bash* is the programming language that is used in the command line; it's actually a synonym for 'Born Again SHell'. The *terminal* is sometimes used to refer to the command line on Macs. Finally, a *CLI* is just an acronym for command line interface, and is often used in the context of an application; for example, uv has a command line interface because you run it on the command line to install packages (`uv add packagename`). +The command line is a way to directly issue text-based commands to a computer one line at a time (as distinct from a graphical user interface, or GUI, that you navigate with a mouse). It goes under many names: shell, bash, terminal, CLI, and command line. These are actually different things but most people tend to use them to mean the same thing most of the time. The _shell_ is the part of an operating system that you interact with but mostly people use shell to mean the command line. _bash_ is the programming language that is used in the command line; it's actually a synonym for 'Born Again SHell'. The _terminal_ is sometimes used to refer to the command line on Macs. Finally, a _CLI_ is just an acronym for command line interface, and is often used in the context of an application; for example, uv has a command line interface because you run it on the command line to install packages (`uv add packagename`). It's worth mentioning that there's a big difference between the command line on UNIX based systems (MacOS and Linux), and on Windows systems. Here, we'll only address the UNIX version. There is a command line on Windows but it's not widely used for coding. If you're on a Windows machine, you can access a UNIX command line using the Windows Subsystem for Linux. ## Why is the command line useful? -The command line has many uses. Graphical user interfaces are, generally, a bit easier to use *but* they're not very repeatable or scalable. Because the command line uses text-based instructions and can be programmed, it is both repeatable and scalable; properties that are very useful for research and analysis. +The command line has many uses. Graphical user interfaces are, generally, a bit easier to use _but_ they're not very repeatable or scalable. Because the command line uses text-based instructions and can be programmed, it is both repeatable and scalable; properties that are very useful for research and analysis. The broad reasons you might use the command line to issue instructions include: -- software functionality: some software *only* has a command line interface +- software functionality: some software _only_ has a command line interface - efficiency: your computer has limited memory, which graphical user interfaces use a lot of—the command line uses less @@ -71,7 +71,7 @@ The flags or options, such as `-n` in the example above, typically begin with a Spaces take on a special role when using the command line. For this reason, it's good practice to avoid spaces in file names. If you need to refer to a filename with spaces in, you’ll need to use quotes or escape the spaces in the file names using a `\`, for example `this is my file.txt` becomes `this\ is\ my\ file.txt` ::: -To run programmes from the command line, all you need is the name of the programme as the command: in fact, commands *are* programmes. The `date` command refers to an actual programme on your computer that you can find. And this also explains a bit of what's going on when you *run a script from the command line* (more on that later). +To run programmes from the command line, all you need is the name of the programme as the command: in fact, commands _are_ programmes. The `date` command refers to an actual programme on your computer that you can find. And this also explains a bit of what's going on when you _run a script from the command line_ (more on that later). Once you've run a few commands, you'll notice that you can't navigate around the command line like you can a text file or Python script. Here are some tips for navigating the command line: @@ -93,20 +93,20 @@ Once you've run a few commands, you'll notice that you can't navigate around the ### Navigating directories -While we're on navigating, it's useful to understand *where* in the computer you are when you open the command line. If you open a terminal pane within VS Code, you will start (by default at least) within the same folder as your project. Starting a terminal instance outside of VS Code will get you a terminal in a root directory for your computer; for example, on a Mac, opening a new terminal window starts you in `/Users/yourusername/`. +While we're on navigating, it's useful to understand _where_ in the computer you are when you open the command line. If you open a terminal pane within VS Code, you will start (by default at least) within the same folder as your project. Starting a terminal instance outside of VS Code will get you a terminal in a root directory for your computer; for example, on a Mac, opening a new terminal window starts you in `/Users/yourusername/`. To find out "where" you are when you open a terminal, you can use the `pwd` command, which stands for "print working directory". -The table below shows some useful commands for moving around your computer using the command line. Note that `cd` accepts a location *relative* to your current directory. +The table below shows some useful commands for moving around your computer using the command line. Note that `cd` accepts a location _relative_ to your current directory. - | Command | What it does | - | --------------------- | ------------------------------------------------------------ | - | `pwd` | Shows current directory | - | `cd` | Change directory command | - | `cd ..` | Go up one level in the directory (`cd ../..` for two levels) | - | `cd ~` | Go to your home directory | - | `cd -` | Go to the previous directory | - | `cd documents/papers` | Go directly to a directory named 'papers' | +| Command | What it does | +| --------------------- | ------------------------------------------------------------ | +| `pwd` | Shows current directory | +| `cd` | Change directory command | +| `cd ..` | Go up one level in the directory (`cd ../..` for two levels) | +| `cd ~` | Go to your home directory | +| `cd -` | Go to the previous directory | +| `cd documents/papers` | Go directly to a directory named 'papers' | ## Using Python on the command line @@ -124,7 +124,7 @@ Say you have a script called `analysis.py`, you can run it with Python on the co uv run python analysis.py ``` -which calls Python as a programme and gives it `analysis.py` as the argument. If you have multiple versions of Python, which you should do if you're following best practice and using a version per project, then you can see *which* version of Python is being used with +which calls Python as a programme and gives it `analysis.py` as the argument. If you have multiple versions of Python, which you should do if you're following best practice and using a version per project, then you can see _which_ version of Python is being used with ```bash which python @@ -134,39 +134,39 @@ which python Now we'll see some useful commands for the terminal. - | Command                                          | What it does | - | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | - | `man ` | Shows a manual for the given command | - | `touch ` | Creates an empty file named `` | - | `code ` | Open a file in VS Code (creating it, if it does not exist) | - | `mkdir ` | creates a new folder called `foldername` | - | `echo ` | Prints `` | - | `cat ` | Print the full contents of `` | - | `head ` | Print the start of a file | - | `tail ` | Print the end of a file | - | `> ` | Redirects output from screen to ``. For example, `echo "Hello World" > hello.txt` | - | `>> ` | Redirects output from screen to the end of ``, ie appends output rather than overwrites it | - | ` | ` | The pipe symbol: uses output from one command as input into another. For example, `head -n 10 data.csv | > hello_world.txt` would write the first 10 lines of data.csv into a file called hello_world.txt | - | `less ` | Print out the contents of a file in paginated form. Use `ctrl+v` and `Alt+v` (or `⌘+v` and `⌥+v` on Mac) to move up and down. Press `q` to quit. | - | `wc -l` | Returns number of lines in input, for example `cat | wc -l`. Use `wc` alone for word count. | - | `sort` | Arrange lines in a file in alphabetical order | - | `uniq` | Remove duplicate lines from input, for example `cat | uniq` or `uniq -d` to show duplicate files | - | `mv` | Move or rename a file; for example, `mv file1 file2` would rename `file1` to `file2` while `mv file1 ~` would move `file1` to the home directory | - | `cp` | Copy a file; for example, `cp file1 file2` would copy `file1` to `file2` while `cp file1 ~` would make a copy of `file1` in the home directory | - | `rm ` | Permanently remove a file | - | `rmdir ` | Permanently remove an empty directory | - | `rm -rf ` | ⚠ Permanently remove everything in a directory ⚠ | - | `grep ` | Search for a given term, for example `cat hello_world.txt | grep world` | - | `ls` | Basically, this means list stuff (files and folders) in the current directory | - | `ls -a` | List stuff in the current directory even if it's hidden | - | `ls -l` | List stuff in a more readable format and show permissions | - | `ls -S` | List stuff by size | - | `file ` | Give information on the file type of `` | - | `find` | Find specific files on your computer, can be piped into other commands for example `find *.md -size +5k -type f | xargs wc -l` will count the number of lines `wc -l` of all files, `-type f`, ending in `.md` that are greater than 5 kilobytes in size, `-size +5k`. | - | `diff -u ` | Show a single summary of the differences between two files. | +| Command                                          | What it does | +| --------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| `man ` | Shows a manual for the given command | +| `touch ` | Creates an empty file named `` | +| `code ` | Open a file in VS Code (creating it, if it does not exist) | +| `mkdir ` | creates a new folder called `foldername` | +| `echo ` | Prints `` | +| `cat ` | Print the full contents of `` | +| `head ` | Print the start of a file | +| `tail ` | Print the end of a file | +| `> ` | Redirects output from screen to ``. For example, `echo "Hello World" > hello.txt` | +| `>> ` | Redirects output from screen to the end of ``, ie appends output rather than overwrites it | +| ` | ` | The pipe symbol: uses output from one command as input into another. For example, `head -n 10 data.csv | > hello_world.txt` would write the first 10 lines of data.csv into a file called hello_world.txt | +| `less ` | Print out the contents of a file in paginated form. Use `ctrl+v` and `Alt+v` (or `⌘+v` and `⌥+v` on Mac) to move up and down. Press `q` to quit. | +| `wc -l` | Returns number of lines in input, for example `cat | wc -l`. Use `wc` alone for word count. | +| `sort` | Arrange lines in a file in alphabetical order | +| `uniq` | Remove duplicate lines from input, for example `cat | uniq`or`uniq -d` to show duplicate files | +| `mv` | Move or rename a file; for example, `mv file1 file2` would rename `file1` to `file2` while `mv file1 ~` would move `file1` to the home directory | +| `cp` | Copy a file; for example, `cp file1 file2` would copy `file1` to `file2` while `cp file1 ~` would make a copy of `file1` in the home directory | +| `rm ` | Permanently remove a file | +| `rmdir ` | Permanently remove an empty directory | +| `rm -rf ` | ⚠ Permanently remove everything in a directory ⚠ | +| `grep ` | Search for a given term, for example `cat hello_world.txt | grep world` | +| `ls` | Basically, this means list stuff (files and folders) in the current directory | +| `ls -a` | List stuff in the current directory even if it's hidden | +| `ls -l` | List stuff in a more readable format and show permissions | +| `ls -S` | List stuff by size | +| `file ` | Give information on the file type of `` | +| `find` | Find specific files on your computer, can be piped into other commands for example `find \*.md -size +5k -type f | xargs wc -l`will count the number of lines`wc -l`of all files,`-type f`, ending in `.md`that are greater than 5 kilobytes in size,`-size +5k`. | +| `diff -u ` | Show a single summary of the differences between two files. | ![More details of the grep command](https://pbs.twimg.com/media/DcPeD_CW0AEkSar?format=jpg&name=small) -*More details of the grep command, by [\@b0rk](https://twitter.com/b0rk).* +_More details of the grep command, by [\@b0rk](https://twitter.com/b0rk)._ You can write for loops in bash (remember, it's a language). The general structure is @@ -187,7 +187,7 @@ A more interesting example is giving the number of lines of text, number of word ```bash for i in $(ls *.csv) -do +do wc $i done ``` @@ -204,7 +204,7 @@ done A couple of new features appeared in the examples above. -`*` is a *wildcard character*, it tells bash to look for anything that ends in ".csv". This is not the only special case; `?` serves a similar purpose of standing in for any character but just *one* character rather than arbitrarily many. If you had a folder with `file1.csv`, `file2.csv`, etc., up to 9, then you could use `file?.csv` to refer to all of them but this would not pick up `file10.csv`. +`*` is a _wildcard character_, it tells bash to look for anything that ends in ".csv". This is not the only special case; `?` serves a similar purpose of standing in for any character but just _one_ character rather than arbitrarily many. If you had a folder with `file1.csv`, `file2.csv`, etc., up to 9, then you could use `file?.csv` to refer to all of them but this would not pick up `file10.csv`. Another special character we've already seen is the curly brace, `{}`. Whenever you have a common substring in a series of commands using curly braces tells the command line to expand what's in them automatically. In an example above, this is used on 1 to 5. But it can also be used in, for example, file names: @@ -272,7 +272,7 @@ You can find more of these special variables [here](https://tldp.org/LDP/abs/htm [**pandoc**](https://pandoc.org/) is absolutely brilliant: if you need to convert files containing text from one format to another, it really is a swiss-army knife. There isn't space here to list the ridiculous number of documents it can convert between, but, importantly, it can translate back and forth between all of the following: markdown, $\LaTeX$, Microsoft Word's docx, OpenOffice's ODT, HTML, and Jupyter Notebook. -It can also write from any of those formats (and more) in one direction *to* PDF, Microsoft Powerpoint, and $\LaTeX$ Beamer. +It can also write from any of those formats (and more) in one direction _to_ PDF, Microsoft Powerpoint, and $\LaTeX$ Beamer. To use **pandoc**, install it following the instructions on the website and then call it like this: @@ -284,9 +284,9 @@ This is an example where the input is a .tex document and the output, `-o`, is a You can get quite fancy with **pandoc**, for example you can translate a whole book's worth of latex into a Word doc complete with a Word style, a bibliography via biblatex, equations, and figures. Nothing can save Word from being painful to use, but **pandoc** certainly helps. -[**eza**](https://eza.rocks/) is an upgrade on the `ls` command. It is designed to be an improved file lister with more features and better defaults. It uses colours to distinguish file types and metadata. Follow the instructions on the website to install it on your operating system. To replace `ls` with `eza`, you can use a terminal *alias*. There's a good guide [available here](https://denisrasulev.medium.com/eza-the-best-ls-command-replacement-9621252323e). +[**eza**](https://eza.rocks/) is an upgrade on the `ls` command. It is designed to be an improved file lister with more features and better defaults. It uses colours to distinguish file types and metadata. Follow the instructions on the website to install it on your operating system. To replace `ls` with `eza`, you can use a terminal _alias_. There's a good guide [available here](https://denisrasulev.medium.com/eza-the-best-ls-command-replacement-9621252323e). -**nano** is a built-in text editor that runs *within* the terminal. This can be really useful if you're working on the cloud (but it's not got the rich features of a GUI-based text editor like VS Code). To open a file using **nano**, the command is `nano file.txt`. Nano displays instructions on how to navigate when it loads up but exiting is the hardest part: when you're done, hit `Ctrl+X`, then `y` to save, and then `enter` to exit. +**nano** is a built-in text editor that runs _within_ the terminal. This can be really useful if you're working on the cloud (but it's not got the rich features of a GUI-based text editor like VS Code). To open a file using **nano**, the command is `nano file.txt`. Nano displays instructions on how to navigate when it loads up but exiting is the hardest part: when you're done, hit `Ctrl+X`, then `y` to save, and then `enter` to exit. [**wget**](https://www.gnu.org/software/wget/) is a command-line utility for downloading files from the internet. It's very simple to use, the syntax is just `wget [options] [url]`. For example, to download the starwars csv file used in this book, the command is diff --git a/functions.ipynb b/functions.ipynb index 70c4fd3..0f2cce3 100644 --- a/functions.ipynb +++ b/functions.ipynb @@ -208,7 +208,7 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (10, 4)
abcd
f64f64f64f64
0.4345330.9990720.3288630.269584
0.00.4799171.00.676026
1.00.00.3837450.381636
0.6584960.4682960.0809160.283739
0.3495880.1646770.6419610.38563
0.3624860.3741330.4753370.024468
0.0426121.1409950.4003250.121255
0.5385220.3190620.5273250.292675
0.5442450.6919370.01.0
0.3524920.2954480.4701810.0
" + "shape: (10, 4)
abcd
f64f64f64f64
0.4736410.3145550.780770.683499
1.00.1921670.8502790.186199
0.4507660.4572280.5157880.0
0.3617790.2195290.2433410.934585
0.028510.00.00.948462
0.4747280.1190681.00.556861
0.1679770.6302890.6544850.727404
0.279770.5057040.9460620.527484
0.4111420.8182760.1750241.0
0.00.1704820.3733090.45716
" ], "text/plain": [ "shape: (10, 4)\n", @@ -217,16 +217,16 @@ "│ --- ┆ --- ┆ --- ┆ --- │\n", "│ f64 ┆ f64 ┆ f64 ┆ f64 │\n", "╞══════════╪══════════╪══════════╪══════════╡\n", - "│ 0.434533 ┆ 0.999072 ┆ 0.328863 ┆ 0.269584 │\n", - "│ 0.0 ┆ 0.479917 ┆ 1.0 ┆ 0.676026 │\n", - "│ 1.0 ┆ 0.0 ┆ 0.383745 ┆ 0.381636 │\n", - "│ 0.658496 ┆ 0.468296 ┆ 0.080916 ┆ 0.283739 │\n", - "│ 0.349588 ┆ 0.164677 ┆ 0.641961 ┆ 0.38563 │\n", - "│ 0.362486 ┆ 0.374133 ┆ 0.475337 ┆ 0.024468 │\n", - "│ 0.042612 ┆ 1.140995 ┆ 0.400325 ┆ 0.121255 │\n", - "│ 0.538522 ┆ 0.319062 ┆ 0.527325 ┆ 0.292675 │\n", - "│ 0.544245 ┆ 0.691937 ┆ 0.0 ┆ 1.0 │\n", - "│ 0.352492 ┆ 0.295448 ┆ 0.470181 ┆ 0.0 │\n", + "│ 0.473641 ┆ 0.314555 ┆ 0.78077 ┆ 0.683499 │\n", + "│ 1.0 ┆ 0.192167 ┆ 0.850279 ┆ 0.186199 │\n", + "│ 0.450766 ┆ 0.457228 ┆ 0.515788 ┆ 0.0 │\n", + "│ 0.361779 ┆ 0.219529 ┆ 0.243341 ┆ 0.934585 │\n", + "│ 0.02851 ┆ 0.0 ┆ 0.0 ┆ 0.948462 │\n", + "│ 0.474728 ┆ 0.119068 ┆ 1.0 ┆ 0.556861 │\n", + "│ 0.167977 ┆ 0.630289 ┆ 0.654485 ┆ 0.727404 │\n", + "│ 0.27977 ┆ 0.505704 ┆ 0.946062 ┆ 0.527484 │\n", + "│ 0.411142 ┆ 0.818276 ┆ 0.175024 ┆ 1.0 │\n", + "│ 0.0 ┆ 0.170482 ┆ 0.373309 ┆ 0.45716 │\n", "└──────────┴──────────┴──────────┴──────────┘" ] }, diff --git a/iteration.ipynb b/iteration.ipynb index 74021f3..fd727cd 100644 --- a/iteration.ipynb +++ b/iteration.ipynb @@ -348,7 +348,7 @@ "[51, 52, 53, 54, 55, 56, 57, 58, 59, 60]" ] }, - "execution_count": 10, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } @@ -536,7 +536,7 @@ "{'Ada': 'Lovelace', 'Adam': 'Smith'}" ] }, - "execution_count": 16, + "execution_count": 41, "metadata": {}, "output_type": "execute_result" } @@ -651,7 +651,7 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (6, 4)
abcd
f64f64f64f64
-0.405242-1.3517780.4098180.975997
1.425904-1.610260.456314-0.371164
-0.555008-0.13544-0.3194540.612513
1.842544-1.7097670.0183770.304892
0.926347-1.3718410.3992051.433585
0.649944-0.011038-0.904321-0.5888
" + "shape: (6, 4)
abcd
f64f64f64f64
-0.871189-0.681725-1.562824-0.560553
-1.2694980.148269-0.217629-0.221637
-0.555918-1.4479222.4705161.641996
0.35463-1.664464-0.0148610.030461
-0.8667982.2567060.968797-0.078444
0.5406780.8744340.286314-0.138539
" ], "text/plain": [ "shape: (6, 4)\n", @@ -660,16 +660,16 @@ "│ --- ┆ --- ┆ --- ┆ --- │\n", "│ f64 ┆ f64 ┆ f64 ┆ f64 │\n", "╞═══════════╪═══════════╪═══════════╪═══════════╡\n", - "│ -0.405242 ┆ -1.351778 ┆ 0.409818 ┆ 0.975997 │\n", - "│ 1.425904 ┆ -1.61026 ┆ 0.456314 ┆ -0.371164 │\n", - "│ -0.555008 ┆ -0.13544 ┆ -0.319454 ┆ 0.612513 │\n", - "│ 1.842544 ┆ -1.709767 ┆ 0.018377 ┆ 0.304892 │\n", - "│ 0.926347 ┆ -1.371841 ┆ 0.399205 ┆ 1.433585 │\n", - "│ 0.649944 ┆ -0.011038 ┆ -0.904321 ┆ -0.5888 │\n", + "│ -0.871189 ┆ -0.681725 ┆ -1.562824 ┆ -0.560553 │\n", + "│ -1.269498 ┆ 0.148269 ┆ -0.217629 ┆ -0.221637 │\n", + "│ -0.555918 ┆ -1.447922 ┆ 2.470516 ┆ 1.641996 │\n", + "│ 0.35463 ┆ -1.664464 ┆ -0.014861 ┆ 0.030461 │\n", + "│ -0.866798 ┆ 2.256706 ┆ 0.968797 ┆ -0.078444 │\n", + "│ 0.540678 ┆ 0.874434 ┆ 0.286314 ┆ -0.138539 │\n", "└───────────┴───────────┴───────────┴───────────┘" ] }, - "execution_count": 18, + "execution_count": 43, "metadata": {}, "output_type": "execute_result" } @@ -706,20 +706,20 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (1, 4)
abcd
f64f64f64f64
0.788146-1.361810.2087910.458703
" + "shape: (1, 4)
abcd
f64f64f64f64
-0.711358-0.2667280.135727-0.108492
" ], "text/plain": [ "shape: (1, 4)\n", - "┌──────────┬──────────┬──────────┬──────────┐\n", - "│ a ┆ b ┆ c ┆ d │\n", - "│ --- ┆ --- ┆ --- ┆ --- │\n", - "│ f64 ┆ f64 ┆ f64 ┆ f64 │\n", - "╞══════════╪══════════╪══════════╪══════════╡\n", - "│ 0.788146 ┆ -1.36181 ┆ 0.208791 ┆ 0.458703 │\n", - "└──────────┴──────────┴──────────┴──────────┘" + "┌───────────┬───────────┬──────────┬───────────┐\n", + "│ a ┆ b ┆ c ┆ d │\n", + "│ --- ┆ --- ┆ --- ┆ --- │\n", + "│ f64 ┆ f64 ┆ f64 ┆ f64 │\n", + "╞═══════════╪═══════════╪══════════╪═══════════╡\n", + "│ -0.711358 ┆ -0.266728 ┆ 0.135727 ┆ -0.108492 │\n", + "└───────────┴───────────┴──────────┴───────────┘" ] }, - "execution_count": 19, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } @@ -744,7 +744,7 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (6, 1)
row_median
f64
0.002288
0.042575
-0.227447
0.161635
0.662776
-0.299919
" + "shape: (6, 1)
row_median
f64
-0.776457
-0.219633
0.543039
0.0078
0.445177
0.413496
" ], "text/plain": [ "shape: (6, 1)\n", @@ -753,16 +753,16 @@ "│ --- │\n", "│ f64 │\n", "╞════════════╡\n", - "│ 0.002288 │\n", - "│ 0.042575 │\n", - "│ -0.227447 │\n", - "│ 0.161635 │\n", - "│ 0.662776 │\n", - "│ -0.299919 │\n", + "│ -0.776457 │\n", + "│ -0.219633 │\n", + "│ 0.543039 │\n", + "│ 0.0078 │\n", + "│ 0.445177 │\n", + "│ 0.413496 │\n", "└────────────┘" ] }, - "execution_count": 20, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } @@ -789,7 +789,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "291 μs ± 1.95 μs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)\n" + "294 μs ± 4.83 μs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)\n" ] } ], @@ -824,7 +824,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "56.7 μs ± 601 ns per loop (mean ± std. dev. of 7 runs, 10,000 loops each)\n" + "56.5 μs ± 929 ns per loop (mean ± std. dev. of 7 runs, 10,000 loops each)\n" ] } ], @@ -888,7 +888,7 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (6, 6)
abcdnew_aresult
f64f64f64f64f64f64
40554.59475840553.64822240555.40981840555.97599740559.594758-7.814557
40556.42590440553.3897440555.45631440554.62883640561.425904-6.288454
40554.44499240554.8645640554.68054640555.61251340559.444992-6.018383
40556.84254440553.29023340555.01837740555.30489240561.842544-5.533329
40555.92634740553.62815940555.39920540556.43358540560.926347-6.492421
40555.64994440554.98896240554.09567940554.411240560.649944-4.104063
" + "shape: (6, 6)
abcdnew_aresult
f64f64f64f64f64f64
40554.12881140554.31827540553.43717640554.43944740559.128811-4.545293
40553.73050240555.14826940554.78237140554.77836340558.730502-5.458877
40554.44408240553.55207840557.47051640556.64199640559.444082-9.030254
40555.3546340553.33553640554.98513940555.03046140560.35463-5.850543
40554.13320240557.25670640555.96879740554.92155640559.133202-4.134134
40555.54067840555.87443440555.28631440554.86146140560.540678-3.426466
" ], "text/plain": [ "shape: (6, 6)\n", @@ -897,16 +897,16 @@ "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", "│ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │\n", "╞══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪═══════════╡\n", - "│ 40554.594758 ┆ 40553.648222 ┆ 40555.409818 ┆ 40555.975997 ┆ 40559.594758 ┆ -7.814557 │\n", - "│ 40556.425904 ┆ 40553.38974 ┆ 40555.456314 ┆ 40554.628836 ┆ 40561.425904 ┆ -6.288454 │\n", - "│ 40554.444992 ┆ 40554.86456 ┆ 40554.680546 ┆ 40555.612513 ┆ 40559.444992 ┆ -6.018383 │\n", - "│ 40556.842544 ┆ 40553.290233 ┆ 40555.018377 ┆ 40555.304892 ┆ 40561.842544 ┆ -5.533329 │\n", - "│ 40555.926347 ┆ 40553.628159 ┆ 40555.399205 ┆ 40556.433585 ┆ 40560.926347 ┆ -6.492421 │\n", - "│ 40555.649944 ┆ 40554.988962 ┆ 40554.095679 ┆ 40554.4112 ┆ 40560.649944 ┆ -4.104063 │\n", + "│ 40554.128811 ┆ 40554.318275 ┆ 40553.437176 ┆ 40554.439447 ┆ 40559.128811 ┆ -4.545293 │\n", + "│ 40553.730502 ┆ 40555.148269 ┆ 40554.782371 ┆ 40554.778363 ┆ 40558.730502 ┆ -5.458877 │\n", + "│ 40554.444082 ┆ 40553.552078 ┆ 40557.470516 ┆ 40556.641996 ┆ 40559.444082 ┆ -9.030254 │\n", + "│ 40555.35463 ┆ 40553.335536 ┆ 40554.985139 ┆ 40555.030461 ┆ 40560.35463 ┆ -5.850543 │\n", + "│ 40554.133202 ┆ 40557.256706 ┆ 40555.968797 ┆ 40554.921556 ┆ 40559.133202 ┆ -4.134134 │\n", + "│ 40555.540678 ┆ 40555.874434 ┆ 40555.286314 ┆ 40554.861461 ┆ 40560.540678 ┆ -3.426466 │\n", "└──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴───────────┘" ] }, - "execution_count": 24, + "execution_count": 49, "metadata": {}, "output_type": "execute_result" } @@ -956,7 +956,7 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (6, 6)
abcdnew_aratio
f64f64f64f64f64f64
40554.59475840553.64822240555.40981840555.97599740559.5947580.999877
40556.42590440553.3897440555.45631440554.62883640561.4259040.999877
40554.44499240554.8645640554.68054640555.61251340559.4449920.999877
40556.84254440553.29023340555.01837740555.30489240561.8425440.999877
40555.92634740553.62815940555.39920540556.43358540560.9263470.999877
40555.64994440554.98896240554.09567940554.411240560.6499440.999877
" + "shape: (6, 6)
abcdnew_aratio
f64f64f64f64f64f64
40554.12881140554.31827540553.43717640554.43944740559.1288110.999877
40553.73050240555.14826940554.78237140554.77836340558.7305020.999877
40554.44408240553.55207840557.47051640556.64199640559.4440820.999877
40555.3546340553.33553640554.98513940555.03046140560.354630.999877
40554.13320240557.25670640555.96879740554.92155640559.1332020.999877
40555.54067840555.87443440555.28631440554.86146140560.5406780.999877
" ], "text/plain": [ "shape: (6, 6)\n", @@ -965,16 +965,16 @@ "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", "│ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │\n", "╞══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════╡\n", - "│ 40554.594758 ┆ 40553.648222 ┆ 40555.409818 ┆ 40555.975997 ┆ 40559.594758 ┆ 0.999877 │\n", - "│ 40556.425904 ┆ 40553.38974 ┆ 40555.456314 ┆ 40554.628836 ┆ 40561.425904 ┆ 0.999877 │\n", - "│ 40554.444992 ┆ 40554.86456 ┆ 40554.680546 ┆ 40555.612513 ┆ 40559.444992 ┆ 0.999877 │\n", - "│ 40556.842544 ┆ 40553.290233 ┆ 40555.018377 ┆ 40555.304892 ┆ 40561.842544 ┆ 0.999877 │\n", - "│ 40555.926347 ┆ 40553.628159 ┆ 40555.399205 ┆ 40556.433585 ┆ 40560.926347 ┆ 0.999877 │\n", - "│ 40555.649944 ┆ 40554.988962 ┆ 40554.095679 ┆ 40554.4112 ┆ 40560.649944 ┆ 0.999877 │\n", + "│ 40554.128811 ┆ 40554.318275 ┆ 40553.437176 ┆ 40554.439447 ┆ 40559.128811 ┆ 0.999877 │\n", + "│ 40553.730502 ┆ 40555.148269 ┆ 40554.782371 ┆ 40554.778363 ┆ 40558.730502 ┆ 0.999877 │\n", + "│ 40554.444082 ┆ 40553.552078 ┆ 40557.470516 ┆ 40556.641996 ┆ 40559.444082 ┆ 0.999877 │\n", + "│ 40555.35463 ┆ 40553.335536 ┆ 40554.985139 ┆ 40555.030461 ┆ 40560.35463 ┆ 0.999877 │\n", + "│ 40554.133202 ┆ 40557.256706 ┆ 40555.968797 ┆ 40554.921556 ┆ 40559.133202 ┆ 0.999877 │\n", + "│ 40555.540678 ┆ 40555.874434 ┆ 40555.286314 ┆ 40554.861461 ┆ 40560.540678 ┆ 0.999877 │\n", "└──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────┘" ] }, - "execution_count": 25, + "execution_count": 50, "metadata": {}, "output_type": "execute_result" } @@ -1008,7 +1008,7 @@ " white-space: pre-wrap;\n", "}\n", "\n", - "shape: (6, 7)
abcdnew_aratioa_gt_0.5
f64f64f64f64f64f64bool
40554.59475840553.64822240555.40981840555.97599740559.5947580.999877true
40556.42590440553.3897440555.45631440554.62883640561.4259040.999877true
40554.44499240554.8645640554.68054640555.61251340559.4449920.999877true
40556.84254440553.29023340555.01837740555.30489240561.8425440.999877true
40555.92634740553.62815940555.39920540556.43358540560.9263470.999877true
40555.64994440554.98896240554.09567940554.411240560.6499440.999877true
" + "shape: (6, 7)
abcdnew_aratioa_gt_0.5
f64f64f64f64f64f64bool
40554.12881140554.31827540553.43717640554.43944740559.1288110.999877true
40553.73050240555.14826940554.78237140554.77836340558.7305020.999877true
40554.44408240553.55207840557.47051640556.64199640559.4440820.999877true
40555.3546340553.33553640554.98513940555.03046140560.354630.999877true
40554.13320240557.25670640555.96879740554.92155640559.1332020.999877true
40555.54067840555.87443440555.28631440554.86146140560.5406780.999877true
" ], "text/plain": [ "shape: (6, 7)\n", @@ -1017,16 +1017,16 @@ "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", "│ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ bool │\n", "╞══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════╪══════════╡\n", - "│ 40554.594758 ┆ 40553.648222 ┆ 40555.409818 ┆ 40555.975997 ┆ 40559.594758 ┆ 0.999877 ┆ true │\n", - "│ 40556.425904 ┆ 40553.38974 ┆ 40555.456314 ┆ 40554.628836 ┆ 40561.425904 ┆ 0.999877 ┆ true │\n", - "│ 40554.444992 ┆ 40554.86456 ┆ 40554.680546 ┆ 40555.612513 ┆ 40559.444992 ┆ 0.999877 ┆ true │\n", - "│ 40556.842544 ┆ 40553.290233 ┆ 40555.018377 ┆ 40555.304892 ┆ 40561.842544 ┆ 0.999877 ┆ true │\n", - "│ 40555.926347 ┆ 40553.628159 ┆ 40555.399205 ┆ 40556.433585 ┆ 40560.926347 ┆ 0.999877 ┆ true │\n", - "│ 40555.649944 ┆ 40554.988962 ┆ 40554.095679 ┆ 40554.4112 ┆ 40560.649944 ┆ 0.999877 ┆ true │\n", + "│ 40554.128811 ┆ 40554.318275 ┆ 40553.437176 ┆ 40554.439447 ┆ 40559.128811 ┆ 0.999877 ┆ true │\n", + "│ 40553.730502 ┆ 40555.148269 ┆ 40554.782371 ┆ 40554.778363 ┆ 40558.730502 ┆ 0.999877 ┆ true │\n", + "│ 40554.444082 ┆ 40553.552078 ┆ 40557.470516 ┆ 40556.641996 ┆ 40559.444082 ┆ 0.999877 ┆ true │\n", + "│ 40555.35463 ┆ 40553.335536 ┆ 40554.985139 ┆ 40555.030461 ┆ 40560.35463 ┆ 0.999877 ┆ true │\n", + "│ 40554.133202 ┆ 40557.256706 ┆ 40555.968797 ┆ 40554.921556 ┆ 40559.133202 ┆ 0.999877 ┆ true │\n", + "│ 40555.540678 ┆ 40555.874434 ┆ 40555.286314 ┆ 40554.861461 ┆ 40560.540678 ┆ 0.999877 ┆ true │\n", "└──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────┴──────────┘" ] }, - "execution_count": 26, + "execution_count": 51, "metadata": {}, "output_type": "execute_result" } From d93ca65a5dc162a87c09fd2400cfe57e1933a1c9 Mon Sep 17 00:00:00 2001 From: Nwabueze Ugoh Date: Tue, 28 Apr 2026 14:05:14 +0100 Subject: [PATCH 32/34] Refactor notebooks to reset execution counts and clear outputs - Updated execution counts to null and cleared outputs in `boolean-data.ipynb`, `categorical-data.ipynb`, `communicate-plots.ipynb`, `data-import.ipynb`, `data-tidy.ipynb`, `data-transform.ipynb`, and other notebooks for a cleaner presentation. - Enhanced overall consistency in notebook formatting by removing previous output displays, ensuring a more streamlined user experience. --- boolean-data.ipynb | 1133 +- categorical-data.ipynb | 357 +- command-line.md | 76 +- communicate-plots.ipynb | 7790 +------------ data-import.ipynb | 324 +- data-tidy.ipynb | 1011 +- data-transform.ipynb | 1484 +-- data-visualise.ipynb | 2520 +---- databases.ipynb | 418 +- dates-and-times.ipynb | 11629 +------------------ exploratory-data-analysis.ipynb | 17751 +----------------------------- functions.ipynb | 247 +- introduction.ipynb | 120 +- iteration.ipynb | 466 +- joins.ipynb | 372 +- missing-values.ipynb | 1468 +-- numbers.ipynb | 7331 +----------- rectangling.ipynb | 357 +- regex.ipynb | 103 +- spreadsheets.ipynb | 416 +- strings.ipynb | 808 +- vis-layers.ipynb | 3900 +------ webscraping-and-apis.ipynb | 498 +- whole-game.ipynb | 108 +- workflow-basics.ipynb | 178 +- workflow-style.ipynb | 34 +- 26 files changed, 1053 insertions(+), 59846 deletions(-) diff --git a/boolean-data.ipynb b/boolean-data.ipynb index 7da9c0b..d389d25 100644 --- a/boolean-data.ipynb +++ b/boolean-data.ipynb @@ -22,21 +22,10 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "7e35b9fc", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "bool_variable = True\n", "bool_variable" @@ -62,21 +51,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "590cd75d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "False" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "not True" ] @@ -108,18 +86,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "51622575", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "False\n" - ] - } - ], + "outputs": [], "source": [ "boolean_condition = 10 == 20\n", "print(boolean_condition)" @@ -145,20 +115,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "0c550daa", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Ada, you achieved a high score.\n", - "You could be called Smith or have a high score\n", - "You are not called Smith and you have a high score\n" - ] - } - ], + "outputs": [], "source": [ "name = \"Ada\"\n", "score = 99\n", @@ -183,19 +143,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "7420e1c1", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "True\n", - "False\n" - ] - } - ], + "outputs": [], "source": [ "name_list = [\"Ada\", \"Adam\"]\n", "name_list_two = [\"Ada\", \"Adam\"]\n", @@ -225,19 +176,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "39caa7be", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "True\n", - "False\n" - ] - } - ], + "outputs": [], "source": [ "name_list = [\"Lovelace\", \"Smith\", \"Hopper\", \"Babbage\"]\n", "\n", @@ -268,18 +210,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "95794e71", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "High score!\n" - ] - } - ], + "outputs": [], "source": [ "score = 98\n", "\n", @@ -309,21 +243,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "cd1cd061", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "a, b = 3, 6\n", "\n", @@ -342,21 +265,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "59638407", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "[x for x in range(12)]" ] @@ -371,21 +283,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "8e8072ea", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[0, 2, 4, 6, 8, 10]" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "[x for x in range(12) if x % 2 == 0]" ] @@ -400,32 +301,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "ec01f460", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[0,\n", - " 'Not divisible by 2',\n", - " 2,\n", - " 'Not divisible by 2',\n", - " 4,\n", - " 'Not divisible by 2',\n", - " 6,\n", - " 'Not divisible by 2',\n", - " 8,\n", - " 'Not divisible by 2',\n", - " 10,\n", - " 'Not divisible by 2']" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "[x if x % 2 == 0 else \"Not divisible by 2\" for x in range(12)]" ] @@ -449,18 +328,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "dc605a93", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Falsy\n" - ] - } - ], + "outputs": [], "source": [ "listy = []\n", "other_listy = [1, 2, 3]\n", @@ -473,18 +344,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "da8fe682", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Truthy\n" - ] - } - ], + "outputs": [], "source": [ "if not (other_listy):\n", " print(\"Falsy\")\n", @@ -502,18 +365,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "d80ba0be", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Falsy\n" - ] - } - ], + "outputs": [], "source": [ "if not 0:\n", " print(\"Falsy\")\n", @@ -523,18 +378,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "1973d44d", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Truthy\n" - ] - } - ], + "outputs": [], "source": [ "if not [0, 0, 0]:\n", " print(\"Falsy\")\n", @@ -552,18 +399,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "62840c4a", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Falsy\n" - ] - } - ], + "outputs": [], "source": [ "if not None:\n", " print(\"Falsy\")\n", @@ -593,21 +432,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "bdcb09a5", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "any([True, False, False])" ] @@ -622,21 +450,10 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "2f666185", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "all([True, True, True, True])" ] @@ -651,21 +468,10 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "78777d9c", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "False" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "all([0, 0, 0, 1])" ] @@ -684,79 +490,10 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "7f338fd7", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
bool_col_1bool_col_2
0FalseTrue
1FalseFalse
2FalseTrue
3TrueFalse
4TrueTrue
\n", - "
" - ], - "text/plain": [ - " bool_col_1 bool_col_2\n", - "0 False True\n", - "1 False False\n", - "2 False True\n", - "3 True False\n", - "4 True True" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import pandas as pd\n", "\n", @@ -779,26 +516,10 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "9cdaec7a", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 True\n", - "1 False\n", - "2 True\n", - "3 True\n", - "4 True\n", - "dtype: bool" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[\"bool_col_1\"] | df[\"bool_col_2\"]" ] @@ -813,23 +534,10 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "89ee3e44", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "bool_col_1 2\n", - "bool_col_2 3\n", - "dtype: int64" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.sum()" ] @@ -844,26 +552,10 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "5e30cee7", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 False\n", - "1 True\n", - "2 False\n", - "3 True\n", - "4 True\n", - "Name: bool_col, dtype: bool" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = pd.DataFrame.from_dict({\"bool_col\": [0, 1, 0, 1, 1]})\n", "df[\"bool_col\"].astype(bool)" @@ -882,127 +574,10 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "9f63005f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
caratcutcolorclaritydepthtablepricexyz
00.23IdealESI261.555.03263.953.982.43
10.21PremiumESI159.861.03263.893.842.31
20.23GoodEVS156.965.03274.054.072.31
30.29PremiumIVS262.458.03344.204.232.63
40.31GoodJSI263.358.03354.344.352.75
\n", - "
" - ], - "text/plain": [ - " carat cut color clarity depth table price x y z\n", - "0 0.23 Ideal E SI2 61.5 55.0 326 3.95 3.98 2.43\n", - "1 0.21 Premium E SI1 59.8 61.0 326 3.89 3.84 2.31\n", - "2 0.23 Good E VS1 56.9 65.0 327 4.05 4.07 2.31\n", - "3 0.29 Premium I VS2 62.4 58.0 334 4.20 4.23 2.63\n", - "4 0.31 Good J SI2 63.3 58.0 335 4.34 4.35 2.75" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "diamonds = pd.read_csv(\n", " \"https://github.com/mwaskom/seaborn-data/raw/master/diamonds.csv\"\n", @@ -1020,220 +595,10 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "7a27f0a0", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
caratcutcolorclaritydepthtablepricexyzexpensive
93380.94PremiumESI160.058.045806.446.373.84True
11800.71IdealGVS162.757.029305.695.733.58True
283340.33PremiumGVS161.358.06664.424.462.72False
139191.23Very GoodJVVS261.257.056656.866.924.22True
264742.37IdealJVS262.257.0160598.528.585.32True
122331.19FairISI164.958.051986.646.554.28True
59570.90Very GoodFSI162.660.039506.106.143.83True
483250.33GoodFSI161.862.05364.404.452.74False
257421.51IdealGVS161.157.0146747.417.384.52True
434010.59Very GoodFSI161.757.014125.395.433.34True
\n", - "
" - ], - "text/plain": [ - " carat cut color clarity depth table price x y z \\\n", - "9338 0.94 Premium E SI1 60.0 58.0 4580 6.44 6.37 3.84 \n", - "1180 0.71 Ideal G VS1 62.7 57.0 2930 5.69 5.73 3.58 \n", - "28334 0.33 Premium G VS1 61.3 58.0 666 4.42 4.46 2.72 \n", - "13919 1.23 Very Good J VVS2 61.2 57.0 5665 6.86 6.92 4.22 \n", - "26474 2.37 Ideal J VS2 62.2 57.0 16059 8.52 8.58 5.32 \n", - "12233 1.19 Fair I SI1 64.9 58.0 5198 6.64 6.55 4.28 \n", - "5957 0.90 Very Good F SI1 62.6 60.0 3950 6.10 6.14 3.83 \n", - "48325 0.33 Good F SI1 61.8 62.0 536 4.40 4.45 2.74 \n", - "25742 1.51 Ideal G VS1 61.1 57.0 14674 7.41 7.38 4.52 \n", - "43401 0.59 Very Good F SI1 61.7 57.0 1412 5.39 5.43 3.34 \n", - "\n", - " expensive \n", - "9338 True \n", - "1180 True \n", - "28334 False \n", - "13919 True \n", - "26474 True \n", - "12233 True \n", - "5957 True \n", - "48325 False \n", - "25742 True \n", - "43401 True " - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "diamonds[\"expensive\"] = diamonds[\"price\"] > 1000\n", "diamonds.sample(10)" @@ -1249,140 +614,10 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "id": "c78a6d47", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
caratcutcolorclaritydepthtablepricexyzexpensive
00.23IdealESI261.555.03263.953.982.43False
10.21PremiumESI159.861.03263.893.842.31False
20.23GoodEVS156.965.03274.054.072.31False
30.29PremiumIVS262.458.03344.204.232.63False
40.31GoodJSI263.358.03354.344.352.75False
\n", - "
" - ], - "text/plain": [ - " carat cut color clarity depth table price x y z \\\n", - "0 0.23 Ideal E SI2 61.5 55.0 326 3.95 3.98 2.43 \n", - "1 0.21 Premium E SI1 59.8 61.0 326 3.89 3.84 2.31 \n", - "2 0.23 Good E VS1 56.9 65.0 327 4.05 4.07 2.31 \n", - "3 0.29 Premium I VS2 62.4 58.0 334 4.20 4.23 2.63 \n", - "4 0.31 Good J SI2 63.3 58.0 335 4.34 4.35 2.75 \n", - "\n", - " expensive \n", - "0 False \n", - "1 False \n", - "2 False \n", - "3 False \n", - "4 False " - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "diamonds.assign(expensive=lambda x: x[\"price\"] > 1000).head()" ] @@ -1397,22 +632,10 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "d12f537d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([False, False, False, False, False, False, False, True, True,\n", - " True, False])" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "diamonds.columns.isin([\"x\", \"y\", \"z\"])" ] @@ -1429,21 +652,10 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "35e73305", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "np.True_" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "diamonds[\"expensive\"].any()" ] @@ -1460,239 +672,10 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "d1bbb0fa", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
caratcutcolorclaritydepthtablepricexyzexpensive
10.21PremiumESI159.861.03263.893.842.31False
80.22FairEVS265.161.03373.873.782.49False
110.23IdealJVS162.856.03403.933.902.46False
120.22PremiumFSI160.461.03423.883.842.33False
140.20PremiumESI260.262.03453.793.752.27False
....................................
539280.79PremiumESI261.458.027566.035.963.68True
539290.71IdealGVS161.456.027565.765.733.53True
539300.71PremiumESI160.555.027565.795.743.49True
539310.71PremiumFSI159.862.027565.745.733.43True
539380.86PremiumHSI261.058.027576.156.123.74True
\n", - "

23423 rows × 11 columns

\n", - "
" - ], - "text/plain": [ - " carat cut color clarity depth table price x y z \\\n", - "1 0.21 Premium E SI1 59.8 61.0 326 3.89 3.84 2.31 \n", - "8 0.22 Fair E VS2 65.1 61.0 337 3.87 3.78 2.49 \n", - "11 0.23 Ideal J VS1 62.8 56.0 340 3.93 3.90 2.46 \n", - "12 0.22 Premium F SI1 60.4 61.0 342 3.88 3.84 2.33 \n", - "14 0.20 Premium E SI2 60.2 62.0 345 3.79 3.75 2.27 \n", - "... ... ... ... ... ... ... ... ... ... ... \n", - "53928 0.79 Premium E SI2 61.4 58.0 2756 6.03 5.96 3.68 \n", - "53929 0.71 Ideal G VS1 61.4 56.0 2756 5.76 5.73 3.53 \n", - "53930 0.71 Premium E SI1 60.5 55.0 2756 5.79 5.74 3.49 \n", - "53931 0.71 Premium F SI1 59.8 62.0 2756 5.74 5.73 3.43 \n", - "53938 0.86 Premium H SI2 61.0 58.0 2757 6.15 6.12 3.74 \n", - "\n", - " expensive \n", - "1 False \n", - "8 False \n", - "11 False \n", - "12 False \n", - "14 False \n", - "... ... \n", - "53928 True \n", - "53929 True \n", - "53930 True \n", - "53931 True \n", - "53938 True \n", - "\n", - "[23423 rows x 11 columns]" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "diamonds[diamonds[\"x\"] > diamonds[\"y\"]]" ] diff --git a/categorical-data.ipynb b/categorical-data.ipynb index 6cc6776..ab6ba30 100644 --- a/categorical-data.ipynb +++ b/categorical-data.ipynb @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "51a55374", "metadata": { "tags": [ @@ -65,26 +65,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "535ef959", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 a\n", - "1 b\n", - "2 c\n", - "3 a\n", - "Name: A, dtype: category\n", - "Categories (3, object): ['a', 'b', 'c']" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", @@ -107,79 +91,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "358c83bb", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
valuegroup
03630 - 39
16160 - 69
28980 - 89
32820 - 29
48180 - 89
\n", - "
" - ], - "text/plain": [ - " value group\n", - "0 36 30 - 39\n", - "1 61 60 - 69\n", - "2 89 80 - 89\n", - "3 28 20 - 29\n", - "4 81 80 - 89" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = pd.DataFrame({\"value\": np.random.randint(0, 100, 20)})\n", "labels = [f\"{i} - {i+9}\" for i in range(0, 100, 10)]\n", @@ -199,22 +114,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "fb389105", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[NaN, 'b', 'c', NaN, 'd', NaN, 'c']\n", - "Categories (3, object): ['b', 'c', 'd']" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "raw_cat = pd.Categorical(\n", " [\"a\", \"b\", \"c\", \"a\", \"d\", \"a\", \"c\"], categories=[\"b\", \"c\", \"d\"]\n", @@ -232,29 +135,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "0497fc16", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 NaN\n", - "1 b\n", - "2 c\n", - "3 NaN\n", - "4 d\n", - "5 NaN\n", - "6 c\n", - "Name: cat_type, dtype: category\n", - "Categories (3, object): ['b', 'c', 'd']" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = pd.DataFrame(raw_cat, columns=[\"cat_type\"])\n", "df[\"cat_type\"]" @@ -278,22 +162,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "f7520d3d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['a', 'b', 'c', 'a', 'd', 'a', 'c']\n", - "Categories (4, object): ['a' < 'b' < 'c' < 'd']" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "ordered_cat = pd.Categorical(\n", " [\"a\", \"b\", \"c\", \"a\", \"d\", \"a\", \"c\"],\n", @@ -317,42 +189,20 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "2caba354", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['b', 'c', 'd'], dtype='object')" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[\"cat_type\"].cat.categories" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "5f1fe093", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "False" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[\"cat_type\"].cat.ordered" ] @@ -377,7 +227,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "097171b8", "metadata": {}, "outputs": [], @@ -395,29 +245,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "4ae6df38", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 NaN\n", - "1 alpha\n", - "2 beta\n", - "3 NaN\n", - "4 gamma\n", - "5 NaN\n", - "6 beta\n", - "Name: cat_type, dtype: category\n", - "Categories (4, object): ['alpha', 'beta', 'gamma', 'delta']" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[\"cat_type\"] = df[\"cat_type\"].cat.add_categories([\"delta\"])\n", "df[\"cat_type\"]" @@ -443,26 +274,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "19f5bdda", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "cat_type\n", - "beta 2\n", - "alpha 1\n", - "gamma 1\n", - "delta 0\n", - "Name: count, dtype: int64" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[\"cat_type\"].value_counts()" ] @@ -479,23 +294,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "f52d5d0d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 beta\n", - "Name: cat_type, dtype: category\n", - "Categories (4, object): ['alpha', 'beta', 'gamma', 'delta']" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[\"cat_type\"].mode()" ] @@ -510,81 +312,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "4d43e94d", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60895/379284818.py:2: FutureWarning: 'M' is deprecated and will be removed in a future version, please use 'ME' instead.\n", - " pd.Series(pd.date_range(\"2015/05/01\", periods=5, freq=\"M\"), dtype=\"category\"),\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
datetime
02015-05-31
12015-06-30
22015-07-31
32015-08-31
42015-09-30
\n", - "
" - ], - "text/plain": [ - " datetime\n", - "0 2015-05-31\n", - "1 2015-06-30\n", - "2 2015-07-31\n", - "3 2015-08-31\n", - "4 2015-09-30" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "time_df = pd.DataFrame(\n", " pd.Series(pd.date_range(\"2015/05/01\", periods=5, freq=\"M\"), dtype=\"category\"),\n", @@ -595,26 +326,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "db697f86", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 5\n", - "1 6\n", - "2 7\n", - "3 8\n", - "4 9\n", - "Name: datetime, dtype: int32" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "time_df[\"datetime\"].dt.month" ] @@ -629,26 +344,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "13e7bc66", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 0\n", - "1 1\n", - "2 2\n", - "3 3\n", - "4 4\n", - "dtype: int8" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "time_df[\"datetime\"].cat.codes" ] diff --git a/command-line.md b/command-line.md index 2cdce9b..f9eeb20 100644 --- a/command-line.md +++ b/command-line.md @@ -99,14 +99,14 @@ To find out "where" you are when you open a terminal, you can use the `pwd` comm The table below shows some useful commands for moving around your computer using the command line. Note that `cd` accepts a location _relative_ to your current directory. -| Command | What it does | -| --------------------- | ------------------------------------------------------------ | -| `pwd` | Shows current directory | -| `cd` | Change directory command | -| `cd ..` | Go up one level in the directory (`cd ../..` for two levels) | -| `cd ~` | Go to your home directory | -| `cd -` | Go to the previous directory | -| `cd documents/papers` | Go directly to a directory named 'papers' | + | Command | What it does | + | --------------------- | ------------------------------------------------------------ | + | `pwd` | Shows current directory | + | `cd` | Change directory command | + | `cd ..` | Go up one level in the directory (`cd ../..` for two levels) | + | `cd ~` | Go to your home directory | + | `cd -` | Go to the previous directory | + | `cd documents/papers` | Go directly to a directory named 'papers' | ## Using Python on the command line @@ -134,36 +134,36 @@ which python Now we'll see some useful commands for the terminal. -| Command                                          | What it does | -| --------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | -| `man ` | Shows a manual for the given command | -| `touch ` | Creates an empty file named `` | -| `code ` | Open a file in VS Code (creating it, if it does not exist) | -| `mkdir ` | creates a new folder called `foldername` | -| `echo ` | Prints `` | -| `cat ` | Print the full contents of `` | -| `head ` | Print the start of a file | -| `tail ` | Print the end of a file | -| `> ` | Redirects output from screen to ``. For example, `echo "Hello World" > hello.txt` | -| `>> ` | Redirects output from screen to the end of ``, ie appends output rather than overwrites it | -| ` | ` | The pipe symbol: uses output from one command as input into another. For example, `head -n 10 data.csv | > hello_world.txt` would write the first 10 lines of data.csv into a file called hello_world.txt | -| `less ` | Print out the contents of a file in paginated form. Use `ctrl+v` and `Alt+v` (or `⌘+v` and `⌥+v` on Mac) to move up and down. Press `q` to quit. | -| `wc -l` | Returns number of lines in input, for example `cat | wc -l`. Use `wc` alone for word count. | -| `sort` | Arrange lines in a file in alphabetical order | -| `uniq` | Remove duplicate lines from input, for example `cat | uniq`or`uniq -d` to show duplicate files | -| `mv` | Move or rename a file; for example, `mv file1 file2` would rename `file1` to `file2` while `mv file1 ~` would move `file1` to the home directory | -| `cp` | Copy a file; for example, `cp file1 file2` would copy `file1` to `file2` while `cp file1 ~` would make a copy of `file1` in the home directory | -| `rm ` | Permanently remove a file | -| `rmdir ` | Permanently remove an empty directory | -| `rm -rf ` | ⚠ Permanently remove everything in a directory ⚠ | -| `grep ` | Search for a given term, for example `cat hello_world.txt | grep world` | -| `ls` | Basically, this means list stuff (files and folders) in the current directory | -| `ls -a` | List stuff in the current directory even if it's hidden | -| `ls -l` | List stuff in a more readable format and show permissions | -| `ls -S` | List stuff by size | -| `file ` | Give information on the file type of `` | -| `find` | Find specific files on your computer, can be piped into other commands for example `find \*.md -size +5k -type f | xargs wc -l`will count the number of lines`wc -l`of all files,`-type f`, ending in `.md`that are greater than 5 kilobytes in size,`-size +5k`. | -| `diff -u ` | Show a single summary of the differences between two files. | + | Command                                          | What it does | + | --------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | + | `man ` | Shows a manual for the given command | + | `touch ` | Creates an empty file named `` | + | `code ` | Open a file in VS Code (creating it, if it does not exist) | + | `mkdir ` | creates a new folder called `foldername` | + | `echo ` | Prints `` | + | `cat ` | Print the full contents of `` | + | `head ` | Print the start of a file | + | `tail ` | Print the end of a file | + | `> ` | Redirects output from screen to ``. For example, `echo "Hello World" > hello.txt` | + | `>> ` | Redirects output from screen to the end of ``, ie appends output rather than overwrites it | + | ` | ` | The pipe symbol: uses output from one command as input into another. For example, `head -n 10 data.csv | > hello_world.txt` would write the first 10 lines of data.csv into a file called hello_world.txt | + | `less ` | Print out the contents of a file in paginated form. Use `ctrl+v` and `Alt+v` (or `⌘+v` and `⌥+v` on Mac) to move up and down. Press `q` to quit. | + | `wc -l` | Returns number of lines in input, for example `cat | wc -l`. Use `wc` alone for word count. | + | `sort` | Arrange lines in a file in alphabetical order | + | `uniq` | Remove duplicate lines from input, for example `cat | uniq`or`uniq -d` to show duplicate files | + | `mv` | Move or rename a file; for example, `mv file1 file2` would rename `file1` to `file2` while `mv file1 ~` would move `file1` to the home directory | + | `cp` | Copy a file; for example, `cp file1 file2` would copy `file1` to `file2` while `cp file1 ~` would make a copy of `file1` in the home directory | + | `rm ` | Permanently remove a file | + | `rmdir ` | Permanently remove an empty directory | + | `rm -rf ` | ⚠ Permanently remove everything in a directory ⚠ | + | `grep ` | Search for a given term, for example `cat hello_world.txt | grep world` | + | `ls` | Basically, this means list stuff (files and folders) in the current directory | + | `ls -a` | List stuff in the current directory even if it's hidden | + | `ls -l` | List stuff in a more readable format and show permissions | + | `ls -S` | List stuff by size | + | `file ` | Give information on the file type of `` | + | `find` | Find specific files on your computer, can be piped into other commands for example `find \*.md -size +5k -type f | xargs wc -l`will count the number of lines`wc -l`of all files,`-type f`, ending in `.md`that are greater than 5 kilobytes in size,`-size +5k`. | + | `diff -u ` | Show a single summary of the differences between two files. | ![More details of the grep command](https://pbs.twimg.com/media/DcPeD_CW0AEkSar?format=jpg&name=small) _More details of the grep command, by [\@b0rk](https://twitter.com/b0rk)._ diff --git a/communicate-plots.ipynb b/communicate-plots.ipynb index f833d44..0f0f567 100644 --- a/communicate-plots.ipynb +++ b/communicate-plots.ipynb @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "78eeea41", "metadata": {}, "outputs": [], @@ -42,51 +42,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "ae4a818a", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "import numpy as np\n", "import polars as pl\n", @@ -107,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "c36b4cd5", "metadata": {}, "outputs": [], @@ -129,132 +88,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "c7574bc6", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point())" ] @@ -279,162 +116,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "24b3513e", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -462,165 +147,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "6489a6bf", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -652,153 +182,14 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "683d547c", "metadata": { "tags": [ "remove-cell" ] }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"cty\", y=\"hwy\", color=\"drv\", shape=\"drv\"))\n", @@ -840,40 +231,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "60826a32", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (3, 4)
drvhwydispldrive_type
strf64f64str
"f"28.162.56"front-wheel drive"
"r"21.05.18"rear-wheel drive"
"4"19.174.0"4-wheel drive"
" - ], - "text/plain": [ - "shape: (3, 4)\n", - "┌─────┬───────┬───────┬───────────────────┐\n", - "│ drv ┆ hwy ┆ displ ┆ drive_type │\n", - "│ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ f64 ┆ f64 ┆ str │\n", - "╞═════╪═══════╪═══════╪═══════════════════╡\n", - "│ f ┆ 28.16 ┆ 2.56 ┆ front-wheel drive │\n", - "│ r ┆ 21.0 ┆ 5.18 ┆ rear-wheel drive │\n", - "│ 4 ┆ 19.17 ┆ 4.0 ┆ 4-wheel drive │\n", - "└─────┴───────┴───────┴───────────────────┘" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "mapping = {\n", " \"4\": \"4-wheel drive\",\n", @@ -902,183 +263,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "6f90c2aa", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\", color=\"drv\"))\n", @@ -1108,254 +296,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "bdcd79bb", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "potential_outliers = mpg.filter(\n", " (pl.col(\"hwy\") > 40) | ((pl.col(\"hwy\") > 20) & (pl.col(\"displ\") > 5))\n", @@ -1392,21 +336,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "d1e2cc3a", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Larger engine sizes tend to\\nhave lower fuel economy.'" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import textwrap\n", "\n", @@ -1417,162 +350,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "e8c09f57", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -1704,137 +485,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "a95604d8", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\", color=\"drv\"))\n", @@ -1855,138 +509,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "1a852304", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\", color=\"drv\"))\n", @@ -2013,7 +539,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "40ac230e", "metadata": {}, "outputs": [], @@ -2033,158 +559,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "1520bb3c", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(diamonds, aes(x=\"cut\", y=\"price\"))\n", @@ -2204,42 +582,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "9d1f993a", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (5, 6)
rownamesnamestartendpartyid
i64stri32i32stri64
1"Eisenhower"19531961"Republican"34
2"Kennedy"19611963"Democratic"35
3"Johnson"19631969"Democratic"36
4"Nixon"19691974"Republican"37
5"Ford"19741977"Republican"38
" - ], - "text/plain": [ - "shape: (5, 6)\n", - "┌──────────┬────────────┬───────┬──────┬────────────┬─────┐\n", - "│ rownames ┆ name ┆ start ┆ end ┆ party ┆ id │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ str ┆ i32 ┆ i32 ┆ str ┆ i64 │\n", - "╞══════════╪════════════╪═══════╪══════╪════════════╪═════╡\n", - "│ 1 ┆ Eisenhower ┆ 1953 ┆ 1961 ┆ Republican ┆ 34 │\n", - "│ 2 ┆ Kennedy ┆ 1961 ┆ 1963 ┆ Democratic ┆ 35 │\n", - "│ 3 ┆ Johnson ┆ 1963 ┆ 1969 ┆ Democratic ┆ 36 │\n", - "│ 4 ┆ Nixon ┆ 1969 ┆ 1974 ┆ Republican ┆ 37 │\n", - "│ 5 ┆ Ford ┆ 1974 ┆ 1977 ┆ Republican ┆ 38 │\n", - "└──────────┴────────────┴───────┴──────┴────────────┴─────┘" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "presidential = pl.read_csv(\n", " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/presidential.csv\",\n", @@ -2255,128 +601,10 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "7d88976d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(presidential, aes(x=\"start\", y=\"id\"))\n", @@ -2403,355 +631,10 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "52d6e86a", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "base = ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point(aes(color=\"class\"))\n", "\n", @@ -2790,130 +673,10 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "2c1d3f8d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(\n", @@ -2934,136 +697,10 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "39b4ef8d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(diamonds, aes(x=\"carat\", y=\"price\"))\n", @@ -3083,272 +720,20 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "f06d7e40", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point(aes(color=\"drv\")))" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "6186b520", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -3377,1235 +762,14 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "bd347524", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:34:54.100390\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# | echo: false\n", "cmaps = [\n", @@ -4707,1126 +871,14 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "d6350c71", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:34:54.622782\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# remove input\n", "for cmap_category, cmap_list in cmaps[3:4]:\n", @@ -5835,1003 +887,14 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "id": "0063a574", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:34:54.717286\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# remove input\n", "for cmap_category, cmap_list in cmaps[2:3]:\n", @@ -6848,138 +911,10 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "9751058d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "mini_presid = presidential.slice(5)\n", "\n", @@ -7007,115 +942,10 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "644fd814", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "prng = np.random.default_rng(1837) # prng=probabilistic random number generator\n", "df_rnd = pl.DataFrame(prng.standard_normal((1000, 2)), schema=[\"x\", \"y\"])\n", @@ -7149,147 +979,10 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "25a29f38", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -7300,147 +993,10 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "id": "42318a59", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "mpg_condition = (\n", " (mpg[\"displ\"] >= 5) & (mpg[\"displ\"] <= 6) & (mpg[\"hwy\"] >= 10) & (mpg[\"hwy\"] <= 25)\n", @@ -7465,153 +1021,10 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "id": "03001d5e", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -7624,153 +1037,10 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "id": "dc3bb833", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -7791,134 +1061,10 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "id": "aee538a8", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "suv = mpg.filter(mpg[\"class\"] == \"suv\")\n", "compact = mpg.filter(mpg[\"class\"] == \"compact\")\n", @@ -7927,134 +1073,10 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "id": "a82c8c23", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(compact, aes(x=\"displ\", y=\"hwy\", color=\"drv\")) + geom_point())" ] @@ -8069,7 +1091,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "id": "db6fce43", "metadata": {}, "outputs": [], @@ -8081,73 +1103,10 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "id": "dd9e6606", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(suv, aes(x=\"displ\", y=\"hwy\", color=\"drv\"))\n", @@ -8160,73 +1119,10 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "id": "bdd8b2c5", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(compact, aes(x=\"displ\", y=\"hwy\", color=\"drv\"))\n", @@ -8276,148 +1172,10 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "id": "0b2364ca", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -8439,159 +1197,10 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "id": "67bfa9c8", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", color=\"drv\"))\n", @@ -8632,228 +1241,10 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "id": "a8081df4", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 40, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "p1 = ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point() + labs(title=\"Plot 1\")\n", "p2 = ggplot(mpg, aes(x=\"drv\", y=\"hwy\")) + geom_boxplot() + labs(title=\"Plot 2\")\n", @@ -8874,21 +1265,10 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "id": "710a6a4f", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'/Users/omagic/Documents/GitHub/python4DSpolars/chart.svg'" - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "ggsave(p1, \"chart.svg\", path=\".\")" ] @@ -8903,25 +1283,17 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "id": "bc831b1b", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ls: stdout: Broken pipe\r\n" - ] - } - ], + "outputs": [], "source": [ "!ls | grep *.svg" ] }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "id": "9cc10ab7", "metadata": { "tags": [ diff --git a/data-import.ipynb b/data-import.ipynb index 9369dde..9852811 100644 --- a/data-import.ipynb +++ b/data-import.ipynb @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "1cf01bda", "metadata": {}, "outputs": [], @@ -54,30 +54,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "eca85c47", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Student ID,Full Name,favourite.food,mealPlan,AGE\r", - "\r\n", - "1,Sunil Huffmann,Strawberry yoghurt,Lunch only,4\r", - "\r\n", - "2,Barclay Lynn,French fries,Lunch only,5\r", - "\r\n", - "3,Jayendra Lyne,N/A,Breakfast and lunch,7\r", - "\r\n", - "4,Leon Rossini,Anchovies,Lunch only,8\r", - "\r\n", - "5,Chidiegwu Dunkel,Pizza,Breakfast and lunch,five\r", - "\r\n", - "6,Güvenç Attila,Ice cream,Lunch only,6" - ] - } - ], + "outputs": [], "source": [ "! cat data/students.csv" ] @@ -92,43 +72,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "232fdfef", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 5)
Student IDFull Namefavourite.foodmealPlanAGE
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only""8"
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""five"
6"Güvenç Attila""Ice cream""Lunch only""6"
" - ], - "text/plain": [ - "shape: (6, 5)\n", - "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", - "│ Student ID ┆ Full Name ┆ favourite.food ┆ mealPlan ┆ AGE │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", - "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", - "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", - "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", - "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", - "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ 8 │\n", - "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ five │\n", - "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", - "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "students = pl.read_csv(\"data/students.csv\")\n", "students" @@ -186,43 +133,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "51969364", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 5)
student_idfull_namefavourite_foodmeal_planage
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only""8"
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""five"
6"Güvenç Attila""Ice cream""Lunch only""6"
" - ], - "text/plain": [ - "shape: (6, 5)\n", - "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", - "│ student_id ┆ full_name ┆ favourite_food ┆ meal_plan ┆ age │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", - "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", - "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", - "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", - "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", - "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ 8 │\n", - "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ five │\n", - "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", - "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from skimpy import clean_columns\n", "\n", @@ -240,40 +154,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "f3c31e4a", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6,)
age
i64
4
5
7
8
5
6
" - ], - "text/plain": [ - "shape: (6,)\n", - "Series: 'age' [i64]\n", - "[\n", - "\t4\n", - "\t5\n", - "\t7\n", - "\t8\n", - "\t5\n", - "\t6\n", - "]" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "students = students.with_columns(pl.col(\"age\").replace(\"five\", 5).cast(pl.Int64))\n", "students[\"age\"]" @@ -297,40 +181,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "678fdd2d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6,)
meal_plan
cat
"Lunch only"
"Lunch only"
"Breakfast and lunch"
"Lunch only"
"Breakfast and lunch"
"Lunch only"
" - ], - "text/plain": [ - "shape: (6,)\n", - "Series: 'meal_plan' [cat]\n", - "[\n", - "\t\"Lunch only\"\n", - "\t\"Lunch only\"\n", - "\t\"Breakfast and lunch\"\n", - "\t\"Lunch only\"\n", - "\t\"Breakfast and lunch\"\n", - "\t\"Lunch only\"\n", - "]" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "students = students.with_columns(pl.col(\"meal_plan\").cast(pl.Categorical))\n", "students[\"meal_plan\"]" @@ -348,25 +202,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "f54108d3", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Schema([('student_id', Int32),\n", - " ('full_name', String),\n", - " ('favourite_food', String),\n", - " ('meal_plan', Categorical(ordering='physical')),\n", - " ('age', Int64)])" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "students = students.cast(\n", " {\"student_id\": pl.Int32, \"full_name\": pl.String, \"age\": pl.Int64}\n", @@ -404,48 +243,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "b80b958b", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (19, 5)
monthyearbranditemn
stri64i64i64i64
"January"2019112343
"January"2019187219
"January"2019118222
"January"2019233331
"January"2019221569
……………
"March"2019136271
"March"2019188203
"March"2019272531
"March"2019287663
"March"2019282886
" - ], - "text/plain": [ - "shape: (19, 5)\n", - "┌─────────┬──────┬───────┬──────┬─────┐\n", - "│ month ┆ year ┆ brand ┆ item ┆ n │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n", - "╞═════════╪══════╪═══════╪══════╪═════╡\n", - "│ January ┆ 2019 ┆ 1 ┆ 1234 ┆ 3 │\n", - "│ January ┆ 2019 ┆ 1 ┆ 8721 ┆ 9 │\n", - "│ January ┆ 2019 ┆ 1 ┆ 1822 ┆ 2 │\n", - "│ January ┆ 2019 ┆ 2 ┆ 3333 ┆ 1 │\n", - "│ January ┆ 2019 ┆ 2 ┆ 2156 ┆ 9 │\n", - "│ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ March ┆ 2019 ┆ 1 ┆ 3627 ┆ 1 │\n", - "│ March ┆ 2019 ┆ 1 ┆ 8820 ┆ 3 │\n", - "│ March ┆ 2019 ┆ 2 ┆ 7253 ┆ 1 │\n", - "│ March ┆ 2019 ┆ 2 ┆ 8766 ┆ 3 │\n", - "│ March ┆ 2019 ┆ 2 ┆ 8288 ┆ 6 │\n", - "└─────────┴──────┴───────┴──────┴─────┘" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "list_of_dataframes = [\n", " pl.read_csv(x)\n", @@ -465,57 +266,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "4a92056c", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "List of csvs is:\n", - "['data/03-sales.csv', 'data/02-sales.csv', 'data/01-sales.csv'] \n", - "\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "shape: (19, 5)
monthyearbranditemn
stri64i64i64i64
"March"2019112343
"March"2019136271
"March"2019188203
"March"2019272531
"March"2019287663
……………
"January"2019118222
"January"2019233331
"January"2019221569
"January"2019239876
"January"2019238276
" - ], - "text/plain": [ - "shape: (19, 5)\n", - "┌─────────┬──────┬───────┬──────┬─────┐\n", - "│ month ┆ year ┆ brand ┆ item ┆ n │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n", - "╞═════════╪══════╪═══════╪══════╪═════╡\n", - "│ March ┆ 2019 ┆ 1 ┆ 1234 ┆ 3 │\n", - "│ March ┆ 2019 ┆ 1 ┆ 3627 ┆ 1 │\n", - "│ March ┆ 2019 ┆ 1 ┆ 8820 ┆ 3 │\n", - "│ March ┆ 2019 ┆ 2 ┆ 7253 ┆ 1 │\n", - "│ March ┆ 2019 ┆ 2 ┆ 8766 ┆ 3 │\n", - "│ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ January ┆ 2019 ┆ 1 ┆ 1822 ┆ 2 │\n", - "│ January ┆ 2019 ┆ 2 ┆ 3333 ┆ 1 │\n", - "│ January ┆ 2019 ┆ 2 ┆ 2156 ┆ 9 │\n", - "│ January ┆ 2019 ┆ 2 ┆ 3987 ┆ 6 │\n", - "│ January ┆ 2019 ┆ 2 ┆ 3827 ┆ 6 │\n", - "└─────────┴──────┴───────┴──────┴─────┘" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import glob\n", "\n", @@ -540,7 +294,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "0bc97749", "metadata": {}, "outputs": [], @@ -558,25 +312,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "542c5223", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Schema([('student_id', Int64),\n", - " ('full_name', String),\n", - " ('favourite_food', String),\n", - " ('meal_plan', String),\n", - " ('age', Int64)])" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pl.read_csv(\"data/students-clean.csv\").schema" ] @@ -597,7 +336,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "16c6ca1b", "metadata": {}, "outputs": [], @@ -615,25 +354,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "bfd5104f", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Schema([('student_id', Int32),\n", - " ('full_name', String),\n", - " ('favourite_food', String),\n", - " ('meal_plan', Categorical(ordering='physical')),\n", - " ('age', Int64)])" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pl.read_ipc(\"data/students-clean.feather\").schema" ] @@ -652,7 +376,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "227f7c50", "metadata": { "tags": [ diff --git a/data-tidy.ipynb b/data-tidy.ipynb index 903b0a0..157081f 100644 --- a/data-tidy.ipynb +++ b/data-tidy.ipynb @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "51a55374", "metadata": { "tags": [ @@ -111,96 +111,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "0f9fbf5a", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - " Unmelted: \n", - " first last job height weight\n", - "0 John Doe Nurse 5.5 130\n", - "1 Mary Bo Economist 6.0 150\n", - "\n", - " Melted: \n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
firstlastquantityvalue
0JohnDoeheight5.5
1MaryBoheight6.0
2JohnDoeweight130.0
3MaryBoweight150.0
\n", - "
" - ], - "text/plain": [ - " first last quantity value\n", - "0 John Doe height 5.5\n", - "1 Mary Bo height 6.0\n", - "2 John Doe weight 130.0\n", - "3 Mary Bo weight 150.0" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import pandas as pd\n", "\n", @@ -235,71 +149,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "bfa121cf", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
country19992000
0Afghanistan745.02666.0
1Brazil37737.080488.0
2China212258.0213766.0
\n", - "
" - ], - "text/plain": [ - " country 1999 2000\n", - "0 Afghanistan 745.0 2666.0\n", - "1 Brazil 37737.0 80488.0\n", - "2 China 212258.0 213766.0" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df_tb = pd.read_parquet(\n", " \"https://github.com/aeturrell/python4DS/raw/refs/heads/main/data/who_tb_cases.parquet\"\n", @@ -317,92 +170,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "dc03ccd9", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
countryyearcases
0Afghanistan1999745.0
1Brazil199937737.0
2China1999212258.0
3Afghanistan20002666.0
4Brazil200080488.0
5China2000213766.0
\n", - "
" - ], - "text/plain": [ - " country year cases\n", - "0 Afghanistan 1999 745.0\n", - "1 Brazil 1999 37737.0\n", - "2 China 1999 212258.0\n", - "3 Afghanistan 2000 2666.0\n", - "4 Brazil 2000 80488.0\n", - "5 China 2000 213766.0" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df_tb.melt(\n", " id_vars=[\"country\"],\n", @@ -432,83 +203,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "293768c1", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
A1970A1980B1970B1980Xid
0ad2.53.22.2900010
1be1.21.3-0.5127471
2cf0.70.1-2.3662972
\n", - "
" - ], - "text/plain": [ - " A1970 A1980 B1970 B1980 X id\n", - "0 a d 2.5 3.2 2.290001 0\n", - "1 b e 1.2 1.3 -0.512747 1\n", - "2 c f 0.7 0.1 -2.366297 2" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import numpy as np\n", "\n", @@ -535,107 +233,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "a9ca2fa8", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
XAB
idyear
019702.290001a2.5
11970-0.512747b1.2
21970-2.366297c0.7
019802.290001d3.2
11980-0.512747e1.3
21980-2.366297f0.1
\n", - "
" - ], - "text/plain": [ - " X A B\n", - "id year \n", - "0 1970 2.290001 a 2.5\n", - "1 1970 -0.512747 b 1.2\n", - "2 1970 -2.366297 c 0.7\n", - "0 1980 2.290001 d 3.2\n", - "1 1980 -0.512747 e 1.3\n", - "2 1980 -2.366297 f 0.1" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pd.wide_to_long(df, stubnames=[\"A\", \"B\"], i=\"id\", j=\"year\")" ] @@ -660,109 +261,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "2b791dd1", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AB
firstsecond
barone1.3129600.427839
two-0.070985-0.738495
bazone1.042258-0.430793
two0.511735-0.782214
fooone-0.847108-1.179077
two1.0210641.015834
quxone-1.194002-0.313362
two2.226642-0.898217
\n", - "
" - ], - "text/plain": [ - " A B\n", - "first second \n", - "bar one 1.312960 0.427839\n", - " two -0.070985 -0.738495\n", - "baz one 1.042258 -0.430793\n", - " two 0.511735 -0.782214\n", - "foo one -0.847108 -1.179077\n", - " two 1.021064 1.015834\n", - "qux one -1.194002 -0.313362\n", - " two 2.226642 -0.898217" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "tuples = list(\n", " zip(\n", @@ -787,38 +289,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "d25eb012", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "first second \n", - "bar one A 1.312960\n", - " B 0.427839\n", - " two A -0.070985\n", - " B -0.738495\n", - "baz one A 1.042258\n", - " B -0.430793\n", - " two A 0.511735\n", - " B -0.782214\n", - "foo one A -0.847108\n", - " B -1.179077\n", - " two A 1.021064\n", - " B 1.015834\n", - "qux one A -1.194002\n", - " B -0.313362\n", - " two A 2.226642\n", - " B -0.898217\n", - "dtype: float64" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = df.stack()\n", "df" @@ -846,95 +320,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "b6742a54", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
firstbarbazfooqux
second
oneA1.3129601.042258-0.847108-1.194002
B0.427839-0.430793-1.179077-0.313362
twoA-0.0709850.5117351.0210642.226642
B-0.738495-0.7822141.015834-0.898217
\n", - "
" - ], - "text/plain": [ - "first bar baz foo qux\n", - "second \n", - "one A 1.312960 1.042258 -0.847108 -1.194002\n", - " B 0.427839 -0.430793 -1.179077 -0.313362\n", - "two A -0.070985 0.511735 1.021064 2.226642\n", - " B -0.738495 -0.782214 1.015834 -0.898217" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.unstack(level=0)" ] @@ -971,91 +360,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "fa612456", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
countryyeartypecount
0Afghanistan1999-01-01cases745
1Afghanistan1999-01-01population19987071
2Afghanistan2000-01-01cases2666
3Afghanistan2000-01-01population20595360
4Brazil1999-01-01cases37737
\n", - "
" - ], - "text/plain": [ - " country year type count\n", - "0 Afghanistan 1999-01-01 cases 745\n", - "1 Afghanistan 1999-01-01 population 19987071\n", - "2 Afghanistan 2000-01-01 cases 2666\n", - "3 Afghanistan 2000-01-01 population 20595360\n", - "4 Brazil 1999-01-01 cases 37737" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df_tb_cp = pd.read_parquet(\n", " \"https://github.com/aeturrell/python4DS/raw/refs/heads/main/data/who_tb_case_and_pop.parquet\"\n", @@ -1081,99 +389,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "e584cf37", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
typecountryyearcasespopulation
0Afghanistan1999-01-0174519987071
1Afghanistan2000-01-01266620595360
2Brazil1999-01-0137737172006362
3Brazil2000-01-0180488174504898
4China1999-01-012122581272915272
5China2000-01-012137661280428583
\n", - "
" - ], - "text/plain": [ - "type country year cases population\n", - "0 Afghanistan 1999-01-01 745 19987071\n", - "1 Afghanistan 2000-01-01 2666 20595360\n", - "2 Brazil 1999-01-01 37737 172006362\n", - "3 Brazil 2000-01-01 80488 174504898\n", - "4 China 1999-01-01 212258 1272915272\n", - "5 China 2000-01-01 213766 1280428583" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pivoted = df_tb_cp.pivot(\n", " index=[\"country\", \"year\"], columns=[\"type\"], values=\"count\"\n", @@ -1191,85 +410,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "97c6d139", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
datevariablevalue
12000-02-29A0.799993
22000-03-31A0.247382
92000-10-31A0.685868
182000-09-30B-0.911245
42000-05-31A0.874574
\n", - "
" - ], - "text/plain": [ - " date variable value\n", - "1 2000-02-29 A 0.799993\n", - "2 2000-03-31 A 0.247382\n", - "9 2000-10-31 A 0.685868\n", - "18 2000-09-30 B -0.911245\n", - "4 2000-05-31 A 0.874574" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import numpy as np\n", "\n", @@ -1295,115 +439,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "04f2bd28", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
variableAB
date
2000-01-31NaNNaN
2000-02-290.7116320.869024
2000-03-310.799993-0.276892
2000-04-300.247382-1.172654
2000-05-31-1.545182-1.452367
2000-06-300.874574-2.377642
2000-07-31-0.735886-2.036017
2000-08-310.0138172.211417
2000-09-30-1.401537-0.896416
2000-10-310.063176-0.911245
\n", - "
" - ], - "text/plain": [ - "variable A B\n", - "date \n", - "2000-01-31 NaN NaN\n", - "2000-02-29 0.711632 0.869024\n", - "2000-03-31 0.799993 -0.276892\n", - "2000-04-30 0.247382 -1.172654\n", - "2000-05-31 -1.545182 -1.452367\n", - "2000-06-30 0.874574 -2.377642\n", - "2000-07-31 -0.735886 -2.036017\n", - "2000-08-31 0.013817 2.211417\n", - "2000-09-30 -1.401537 -0.896416\n", - "2000-10-31 0.063176 -0.911245" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.pivot(index=\"date\", columns=\"variable\", values=\"value\").shift(1)" ] diff --git a/data-transform.ipynb b/data-transform.ipynb index 03eed1b..8dfc2c1 100644 --- a/data-transform.ipynb +++ b/data-transform.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "04dcb195", "metadata": {}, "outputs": [], @@ -44,21 +44,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "09eb2e2e", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'1.19.0'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pl.__version__" ] @@ -77,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "eff283e8", "metadata": {}, "outputs": [], @@ -108,42 +97,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "39f99d76", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (5, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64str
201311517515283081911"UA"1545"N14228""EWR""IAH"2271400515"2013-01-01T10:00:00Z"
201311533529485083020"UA"1714"N24211""LGA""IAH"2271416529"2013-01-01T10:00:00Z"
201311542540292385033"AA"1141"N619AA""JFK""MIA"1601089540"2013-01-01T10:00:00Z"
201311544545-110041022-18"B6"725"N804JB""JFK""BQN"1831576545"2013-01-01T10:00:00Z"
201311554600-6812837-25"DL"461"N668DN""LGA""ATL"11676260"2013-01-01T11:00:00Z"
" - ], - "text/plain": [ - "shape: (5, 19)\n", - "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬──────────────────────┐\n", - "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", - "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪══════════════════════╡\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 517 ┆ … ┆ 1400 ┆ 5 ┆ 15 ┆ 2013-01-01T10:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 533 ┆ … ┆ 1416 ┆ 5 ┆ 29 ┆ 2013-01-01T10:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 542 ┆ … ┆ 1089 ┆ 5 ┆ 40 ┆ 2013-01-01T10:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 544 ┆ … ┆ 1576 ┆ 5 ┆ 45 ┆ 2013-01-01T10:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 554 ┆ … ┆ 762 ┆ 6 ┆ 0 ┆ 2013-01-01T11:00:00Z │\n", - "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴──────────────────────┘" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights.head()" ] @@ -158,39 +115,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "95dea97b", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Rows: 336776\n", - "Columns: 19\n", - "$ year 2013, 2013, 2013, 2013, 2013\n", - "$ month 1, 1, 1, 1, 1\n", - "$ day 1, 1, 1, 1, 1\n", - "$ dep_time 517, 533, 542, 544, 554\n", - "$ sched_dep_time 515, 529, 540, 545, 600\n", - "$ dep_delay 2, 4, 2, -1, -6\n", - "$ arr_time 830, 850, 923, 1004, 812\n", - "$ sched_arr_time 819, 830, 850, 1022, 837\n", - "$ arr_delay 11, 20, 33, -18, -25\n", - "$ carrier 'UA', 'UA', 'AA', 'B6', 'DL'\n", - "$ flight 1545, 1714, 1141, 725, 461\n", - "$ tailnum 'N14228', 'N24211', 'N619AA', 'N804JB', 'N668DN'\n", - "$ origin 'EWR', 'LGA', 'JFK', 'JFK', 'LGA'\n", - "$ dest 'IAH', 'IAH', 'MIA', 'BQN', 'ATL'\n", - "$ air_time 227, 227, 160, 183, 116\n", - "$ distance 1400, 1416, 1089, 1576, 762\n", - "$ hour 5, 5, 5, 5, 6\n", - "$ minute 15, 29, 40, 45, 0\n", - "$ time_hour '2013-01-01T10:00:00Z', '2013-01-01T10:00:00Z', '2013-01-01T10:00:00Z', '2013-01-01T10:00:00Z', '2013-01-01T11:00:00Z'\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "flights.glimpse(max_items_per_column=5)" ] @@ -221,93 +149,20 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "ffb275b0", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (336_776,)
time_hour
str
"2013-01-01T10:00:00Z"
"2013-01-01T10:00:00Z"
"2013-01-01T10:00:00Z"
"2013-01-01T10:00:00Z"
"2013-01-01T11:00:00Z"
…
"2013-09-30T18:00:00Z"
"2013-10-01T02:00:00Z"
"2013-09-30T16:00:00Z"
"2013-09-30T15:00:00Z"
"2013-09-30T12:00:00Z"
" - ], - "text/plain": [ - "shape: (336_776,)\n", - "Series: 'time_hour' [str]\n", - "[\n", - "\t\"2013-01-01T10:00:00Z\"\n", - "\t\"2013-01-01T10:00:00Z\"\n", - "\t\"2013-01-01T10:00:00Z\"\n", - "\t\"2013-01-01T10:00:00Z\"\n", - "\t\"2013-01-01T11:00:00Z\"\n", - "\t…\n", - "\t\"2013-09-30T18:00:00Z\"\n", - "\t\"2013-10-01T02:00:00Z\"\n", - "\t\"2013-09-30T16:00:00Z\"\n", - "\t\"2013-09-30T15:00:00Z\"\n", - "\t\"2013-09-30T12:00:00Z\"\n", - "]" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights.get_column(\"time_hour\")" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "88a8b983", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (336_776, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64datetime[μs, UTC]
201311517515283081911"UA"1545"N14228""EWR""IAH"22714005152013-01-01 10:00:00 UTC
201311533529485083020"UA"1714"N24211""LGA""IAH"22714165292013-01-01 10:00:00 UTC
201311542540292385033"AA"1141"N619AA""JFK""MIA"16010895402013-01-01 10:00:00 UTC
201311544545-110041022-18"B6"725"N804JB""JFK""BQN"18315765452013-01-01 10:00:00 UTC
201311554600-6812837-25"DL"461"N668DN""LGA""ATL"116762602013-01-01 11:00:00 UTC
…………………………………………………
2013930null1455nullnull1634null"9E"3393null"JFK""DCA"null21314552013-09-30 18:00:00 UTC
2013930null2200nullnull2312null"9E"3525null"LGA""SYR"null1982202013-10-01 02:00:00 UTC
2013930null1210nullnull1330null"MQ"3461"N535MQ""LGA""BNA"null76412102013-09-30 16:00:00 UTC
2013930null1159nullnull1344null"MQ"3572"N511MQ""LGA""CLE"null41911592013-09-30 15:00:00 UTC
2013930null840nullnull1020null"MQ"3531"N839MQ""LGA""RDU"null4318402013-09-30 12:00:00 UTC
" - ], - "text/plain": [ - "shape: (336_776, 19)\n", - "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬─────────────────────────┐\n", - "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ datetime[μs, UTC] │\n", - "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪═════════════════════════╡\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 517 ┆ … ┆ 1400 ┆ 5 ┆ 15 ┆ 2013-01-01 10:00:00 UTC │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 533 ┆ … ┆ 1416 ┆ 5 ┆ 29 ┆ 2013-01-01 10:00:00 UTC │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 542 ┆ … ┆ 1089 ┆ 5 ┆ 40 ┆ 2013-01-01 10:00:00 UTC │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 544 ┆ … ┆ 1576 ┆ 5 ┆ 45 ┆ 2013-01-01 10:00:00 UTC │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 554 ┆ … ┆ 762 ┆ 6 ┆ 0 ┆ 2013-01-01 11:00:00 UTC │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 213 ┆ 14 ┆ 55 ┆ 2013-09-30 18:00:00 UTC │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 198 ┆ 22 ┆ 0 ┆ 2013-10-01 02:00:00 UTC │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 764 ┆ 12 ┆ 10 ┆ 2013-09-30 16:00:00 UTC │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 419 ┆ 11 ┆ 59 ┆ 2013-09-30 15:00:00 UTC │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 431 ┆ 8 ┆ 40 ┆ 2013-09-30 12:00:00 UTC │\n", - "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴─────────────────────────┘" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights.with_columns(pl.col(\"time_hour\").str.to_datetime())" ] @@ -332,41 +187,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "1b6bd8b1", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (4, 5)
col0col1col2col3col4
i64i64i64strstr
000"a""alpha"
000"b""gamma"
000"b""gamma"
000"a""gamma"
" - ], - "text/plain": [ - "shape: (4, 5)\n", - "┌──────┬──────┬──────┬──────┬───────┐\n", - "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ str ┆ str │\n", - "╞══════╪══════╪══════╪══════╪═══════╡\n", - "│ 0 ┆ 0 ┆ 0 ┆ a ┆ alpha │\n", - "│ 0 ┆ 0 ┆ 0 ┆ b ┆ gamma │\n", - "│ 0 ┆ 0 ┆ 0 ┆ b ┆ gamma │\n", - "│ 0 ┆ 0 ┆ 0 ┆ a ┆ gamma │\n", - "└──────┴──────┴──────┴──────┴───────┘" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = pl.DataFrame(\n", " data={\n", @@ -405,48 +229,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "cb114649", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (365, 4)
yearmonthdayarr_delay
i64i64i64f64
2013102-19.47619
201342630.380952
20131129-25.384615
2013430-9.52381
201381614.05
…………
20135151.095238
20137912.714286
2013531-6.52381
201311235.375
2013416-1.736842
" - ], - "text/plain": [ - "shape: (365, 4)\n", - "┌──────┬───────┬─────┬────────────┐\n", - "│ year ┆ month ┆ day ┆ arr_delay │\n", - "│ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ f64 │\n", - "╞══════╪═══════╪═════╪════════════╡\n", - "│ 2013 ┆ 10 ┆ 2 ┆ -19.47619 │\n", - "│ 2013 ┆ 4 ┆ 26 ┆ 30.380952 │\n", - "│ 2013 ┆ 11 ┆ 29 ┆ -25.384615 │\n", - "│ 2013 ┆ 4 ┆ 30 ┆ -9.52381 │\n", - "│ 2013 ┆ 8 ┆ 16 ┆ 14.05 │\n", - "│ … ┆ … ┆ … ┆ … │\n", - "│ 2013 ┆ 5 ┆ 15 ┆ 1.095238 │\n", - "│ 2013 ┆ 7 ┆ 9 ┆ 12.714286 │\n", - "│ 2013 ┆ 5 ┆ 31 ┆ -6.52381 │\n", - "│ 2013 ┆ 11 ┆ 23 ┆ 5.375 │\n", - "│ 2013 ┆ 4 ┆ 16 ┆ -1.736842 │\n", - "└──────┴───────┴─────┴────────────┘" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights.filter(pl.col(\"dest\") == \"IAH\").group_by([\"year\", \"month\", \"day\"]).agg(\n", " pl.col(\"arr_delay\").mean()\n", @@ -481,43 +267,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "3958ddb5", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 7)
col0col1col2col3col4col5col6
i64i64i64i64i64i64str
012345"apple"
67891011"orange"
121314151617"pineapple"
181920212223"mango"
242526272829"kiwi"
303132333435"lemon"
" - ], - "text/plain": [ - "shape: (6, 7)\n", - "┌──────┬──────┬──────┬──────┬──────┬──────┬───────────┐\n", - "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 ┆ col6 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", - "╞══════╪══════╪══════╪══════╪══════╪══════╪═══════════╡\n", - "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ 4 ┆ 5 ┆ apple │\n", - "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ 10 ┆ 11 ┆ orange │\n", - "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ 16 ┆ 17 ┆ pineapple │\n", - "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ 22 ┆ 23 ┆ mango │\n", - "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 ┆ kiwi │\n", - "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ 34 ┆ 35 ┆ lemon │\n", - "└──────┴──────┴──────┴──────┴──────┴──────┴───────────┘" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import numpy as np\n", "\n", @@ -546,21 +299,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "0c2faf83", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(24, 25, 26, 27, 28, 29, 'kiwi')" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Gets the first row of the DataFrame\n", "df.row(0)\n", @@ -579,21 +321,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "9d34599b", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(18, 19, 20, 21, 22, 23, 'mango')" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.row(by_predicate=pl.col(\"col6\") == \"mango\")" ] @@ -608,27 +339,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "9bf6313b", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'col0': 18,\n", - " 'col1': 19,\n", - " 'col2': 20,\n", - " 'col3': 21,\n", - " 'col4': 22,\n", - " 'col5': 23,\n", - " 'col6': 'mango'}" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Get the first row of the DataFrame as a dictionary\n", "df.row(0, named=True)\n", @@ -647,39 +361,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "f7e8e892", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (2, 7)
col0col1col2col3col4col5col6
i64i64i64i64i64i64str
242526272829"kiwi"
303132333435"lemon"
" - ], - "text/plain": [ - "shape: (2, 7)\n", - "┌──────┬──────┬──────┬──────┬──────┬──────┬───────┐\n", - "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 ┆ col6 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", - "╞══════╪══════╪══════╪══════╪══════╪══════╪═══════╡\n", - "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 ┆ kiwi │\n", - "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ 34 ┆ 35 ┆ lemon │\n", - "└──────┴──────┴──────┴──────┴──────┴──────┴───────┘" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.slice(-2, 2)" ] @@ -696,39 +381,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "54a9d2b1", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (2, 7)
col0col1col2col3col4col5col6
i64i64i64i64i64i64str
121314151617"pineapple"
242526272829"kiwi"
" - ], - "text/plain": [ - "shape: (2, 7)\n", - "┌──────┬──────┬──────┬──────┬──────┬──────┬───────────┐\n", - "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 ┆ col6 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", - "╞══════╪══════╪══════╪══════╪══════╪══════╪═══════════╡\n", - "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ 16 ┆ 17 ┆ pineapple │\n", - "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 ┆ kiwi │\n", - "└──────┴──────┴──────┴──────┴──────┴──────┴───────────┘" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.filter((pl.col(\"col6\") == \"kiwi\") | (pl.col(\"col6\") == \"pineapple\"))" ] @@ -743,41 +399,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "7849a962", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (4, 7)
col0col1col2col3col4col5col6
i64i64i64i64i64i64str
121314151617"pineapple"
181920212223"mango"
242526272829"kiwi"
303132333435"lemon"
" - ], - "text/plain": [ - "shape: (4, 7)\n", - "┌──────┬──────┬──────┬──────┬──────┬──────┬───────────┐\n", - "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 ┆ col6 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", - "╞══════╪══════╪══════╪══════╪══════╪══════╪═══════════╡\n", - "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ 16 ┆ 17 ┆ pineapple │\n", - "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ 22 ┆ 23 ┆ mango │\n", - "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 ┆ kiwi │\n", - "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ 34 ┆ 35 ┆ lemon │\n", - "└──────┴──────┴──────┴──────┴──────┴──────┴───────────┘" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.filter(pl.col(\"col0\") > 6)" ] @@ -792,48 +417,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "c6dd919f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (4_334, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64str
201311517515283081911"UA"1545"N14228""EWR""IAH"2271400515"2013-01-01T10:00:00Z"
201311533529485083020"UA"1714"N24211""LGA""IAH"2271416529"2013-01-01T10:00:00Z"
201311542540292385033"AA"1141"N619AA""JFK""MIA"1601089540"2013-01-01T10:00:00Z"
201311544545-110041022-18"B6"725"N804JB""JFK""BQN"1831576545"2013-01-01T10:00:00Z"
201311554600-6812837-25"DL"461"N668DN""LGA""ATL"11676260"2013-01-01T11:00:00Z"
…………………………………………………
20131523552359-4425442-17"B6"707"N583JB""JFK""SJU"19315982359"2013-01-06T04:00:00Z"
20131523572359-2432437-5"B6"727"N649JB""JFK""BQN"19515762359"2013-01-06T04:00:00Z"
201315null1400nullnull1518null"EV"5712"N827AS""JFK""IAD"null228140"2013-01-05T19:00:00Z"
201315null840nullnull1001null"9E"3422null"JFK""BOS"null187840"2013-01-05T13:00:00Z"
201315null1430nullnull1735null"AA"883"N544AA""EWR""DFW"null13721430"2013-01-05T19:00:00Z"
" - ], - "text/plain": [ - "shape: (4_334, 19)\n", - "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬──────────────────────┐\n", - "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", - "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪══════════════════════╡\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 517 ┆ … ┆ 1400 ┆ 5 ┆ 15 ┆ 2013-01-01T10:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 533 ┆ … ┆ 1416 ┆ 5 ┆ 29 ┆ 2013-01-01T10:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 542 ┆ … ┆ 1089 ┆ 5 ┆ 40 ┆ 2013-01-01T10:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 544 ┆ … ┆ 1576 ┆ 5 ┆ 45 ┆ 2013-01-01T10:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 554 ┆ … ┆ 762 ┆ 6 ┆ 0 ┆ 2013-01-01T11:00:00Z │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 2013 ┆ 1 ┆ 5 ┆ 2355 ┆ … ┆ 1598 ┆ 23 ┆ 59 ┆ 2013-01-06T04:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 5 ┆ 2357 ┆ … ┆ 1576 ┆ 23 ┆ 59 ┆ 2013-01-06T04:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 5 ┆ null ┆ … ┆ 228 ┆ 14 ┆ 0 ┆ 2013-01-05T19:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 5 ┆ null ┆ … ┆ 187 ┆ 8 ┆ 40 ┆ 2013-01-05T13:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 5 ┆ null ┆ … ┆ 1372 ┆ 14 ┆ 30 ┆ 2013-01-05T19:00:00Z │\n", - "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴──────────────────────┘" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Flights that departed on January 1\n", "flights.filter((pl.col(\"month\") == 1) & (pl.col(\"day\") <= 5))" @@ -859,96 +446,20 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "395c9c62", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (336_776, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64str
201311null1630nullnull1815null"EV"4308"N18120""EWR""RDU"null4161630"2013-01-01T21:00:00Z"
201311null1935nullnull2240null"AA"791"N3EHAA""LGA""DFW"null13891935"2013-01-02T00:00:00Z"
201311null1500nullnull1825null"AA"1925"N3EVAA""LGA""MIA"null1096150"2013-01-01T20:00:00Z"
201311null600nullnull901null"B6"125"N618JB""JFK""FLL"null106960"2013-01-01T11:00:00Z"
201312null1540nullnull1747null"EV"4352"N10575""EWR""CVG"null5691540"2013-01-02T20:00:00Z"
…………………………………………………
201372824002359141134427"B6"1503"N503JB""JFK""SJU"20415982359"2013-07-29T03:00:00Z"
20138102400224575110169"B6"234"N328JB""JFK""BTV"532662245"2013-08-11T02:00:00Z"
20138202400235913543504"B6"745"N708JB""JFK""PSE"20116172359"2013-08-21T03:00:00Z"
20139224002359141134031"B6"839"N828JB""JFK""BQN"21715762359"2013-09-03T03:00:00Z"
2013912240020002402032230213"DL"1147"N910DE""LGA""ATL"101762200"2013-09-13T00:00:00Z"
" - ], - "text/plain": [ - "shape: (336_776, 19)\n", - "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬──────────────────────┐\n", - "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", - "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪══════════════════════╡\n", - "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 416 ┆ 16 ┆ 30 ┆ 2013-01-01T21:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1389 ┆ 19 ┆ 35 ┆ 2013-01-02T00:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1096 ┆ 15 ┆ 0 ┆ 2013-01-01T20:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1069 ┆ 6 ┆ 0 ┆ 2013-01-01T11:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 2 ┆ null ┆ … ┆ 569 ┆ 15 ┆ 40 ┆ 2013-01-02T20:00:00Z │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 2013 ┆ 7 ┆ 28 ┆ 2400 ┆ … ┆ 1598 ┆ 23 ┆ 59 ┆ 2013-07-29T03:00:00Z │\n", - "│ 2013 ┆ 8 ┆ 10 ┆ 2400 ┆ … ┆ 266 ┆ 22 ┆ 45 ┆ 2013-08-11T02:00:00Z │\n", - "│ 2013 ┆ 8 ┆ 20 ┆ 2400 ┆ … ┆ 1617 ┆ 23 ┆ 59 ┆ 2013-08-21T03:00:00Z │\n", - "│ 2013 ┆ 9 ┆ 2 ┆ 2400 ┆ … ┆ 1576 ┆ 23 ┆ 59 ┆ 2013-09-03T03:00:00Z │\n", - "│ 2013 ┆ 9 ┆ 12 ┆ 2400 ┆ … ┆ 762 ┆ 20 ┆ 0 ┆ 2013-09-13T00:00:00Z │\n", - "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴──────────────────────┘" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights.sort(\"dep_time\")" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "9d5f4270", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (336_776, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64str
201311null1935nullnull2240null"AA"791"N3EHAA""LGA""DFW"null13891935"2013-01-02T00:00:00Z"
201311null1500nullnull1825null"AA"1925"N3EVAA""LGA""MIA"null1096150"2013-01-01T20:00:00Z"
201311null1630nullnull1815null"EV"4308"N18120""EWR""RDU"null4161630"2013-01-01T21:00:00Z"
201311null600nullnull901null"B6"125"N618JB""JFK""FLL"null106960"2013-01-01T11:00:00Z"
201311517515283081911"UA"1545"N14228""EWR""IAH"2271400515"2013-01-01T10:00:00Z"
…………………………………………………
20131231232122503146838"B6"2002"N179JB""JFK""BUF"663012250"2014-01-01T03:00:00Z"
2013123123282330-24124093"B6"1389"N651JB""EWR""SJU"19816082330"2014-01-01T04:00:00Z"
20131231233222454758355"B6"486"N334JB""JFK""ROC"602642245"2014-01-01T03:00:00Z"
2013123123552359-4430440-10"B6"1503"N509JB""JFK""SJU"19515982359"2014-01-01T04:00:00Z"
2013123123562359-3436445-9"B6"745"N665JB""JFK""PSE"20016172359"2014-01-01T04:00:00Z"
" - ], - "text/plain": [ - "shape: (336_776, 19)\n", - "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬──────────────────────┐\n", - "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", - "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪══════════════════════╡\n", - "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1389 ┆ 19 ┆ 35 ┆ 2013-01-02T00:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1096 ┆ 15 ┆ 0 ┆ 2013-01-01T20:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 416 ┆ 16 ┆ 30 ┆ 2013-01-01T21:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1069 ┆ 6 ┆ 0 ┆ 2013-01-01T11:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 517 ┆ … ┆ 1400 ┆ 5 ┆ 15 ┆ 2013-01-01T10:00:00Z │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 2013 ┆ 12 ┆ 31 ┆ 2321 ┆ … ┆ 301 ┆ 22 ┆ 50 ┆ 2014-01-01T03:00:00Z │\n", - "│ 2013 ┆ 12 ┆ 31 ┆ 2328 ┆ … ┆ 1608 ┆ 23 ┆ 30 ┆ 2014-01-01T04:00:00Z │\n", - "│ 2013 ┆ 12 ┆ 31 ┆ 2332 ┆ … ┆ 264 ┆ 22 ┆ 45 ┆ 2014-01-01T03:00:00Z │\n", - "│ 2013 ┆ 12 ┆ 31 ┆ 2355 ┆ … ┆ 1598 ┆ 23 ┆ 59 ┆ 2014-01-01T04:00:00Z │\n", - "│ 2013 ┆ 12 ┆ 31 ┆ 2356 ┆ … ┆ 1617 ┆ 23 ┆ 59 ┆ 2014-01-01T04:00:00Z │\n", - "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴──────────────────────┘" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Sort by multiple columns by passing a list of columns.\n", "flights.sort([\"year\", \"month\", \"day\", \"dep_time\"])\n", @@ -968,96 +479,20 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "483acdc1", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (336_776, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64str
201311null1630nullnull1815null"EV"4308"N18120""EWR""RDU"null4161630"2013-01-01T21:00:00Z"
201311null1935nullnull2240null"AA"791"N3EHAA""LGA""DFW"null13891935"2013-01-02T00:00:00Z"
201311null1500nullnull1825null"AA"1925"N3EVAA""LGA""MIA"null1096150"2013-01-01T20:00:00Z"
201311null600nullnull901null"B6"125"N618JB""JFK""FLL"null106960"2013-01-01T11:00:00Z"
201312null1540nullnull1747null"EV"4352"N10575""EWR""CVG"null5691540"2013-01-02T20:00:00Z"
…………………………………………………
201312917031730-2719471957-10"F9"837"N208FR""LGA""DEN"25016201730"2013-01-29T22:00:00Z"
201311119001930-3022332243-10"DL"1435"N934DL""LGA""TPA"13910101930"2013-01-12T00:00:00Z"
2013111014081440-3215491559-10"EV"5713"N825AS""LGA""IAD"522291440"2013-11-10T19:00:00Z"
20132320222055-3322402338-58"DL"1715"N612DL""LGA""MSY"16211832055"2013-02-04T01:00:00Z"
201312720402123-4340235248"B6"97"N592JB""JFK""DEN"26516262123"2013-12-08T02:00:00Z"
" - ], - "text/plain": [ - "shape: (336_776, 19)\n", - "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬──────────────────────┐\n", - "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", - "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪══════════════════════╡\n", - "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 416 ┆ 16 ┆ 30 ┆ 2013-01-01T21:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1389 ┆ 19 ┆ 35 ┆ 2013-01-02T00:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1096 ┆ 15 ┆ 0 ┆ 2013-01-01T20:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ null ┆ … ┆ 1069 ┆ 6 ┆ 0 ┆ 2013-01-01T11:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 2 ┆ null ┆ … ┆ 569 ┆ 15 ┆ 40 ┆ 2013-01-02T20:00:00Z │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 2013 ┆ 1 ┆ 29 ┆ 1703 ┆ … ┆ 1620 ┆ 17 ┆ 30 ┆ 2013-01-29T22:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 11 ┆ 1900 ┆ … ┆ 1010 ┆ 19 ┆ 30 ┆ 2013-01-12T00:00:00Z │\n", - "│ 2013 ┆ 11 ┆ 10 ┆ 1408 ┆ … ┆ 229 ┆ 14 ┆ 40 ┆ 2013-11-10T19:00:00Z │\n", - "│ 2013 ┆ 2 ┆ 3 ┆ 2022 ┆ … ┆ 1183 ┆ 20 ┆ 55 ┆ 2013-02-04T01:00:00Z │\n", - "│ 2013 ┆ 12 ┆ 7 ┆ 2040 ┆ … ┆ 1626 ┆ 21 ┆ 23 ┆ 2013-12-08T02:00:00Z │\n", - "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴──────────────────────┘" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights.sort(\"dep_delay\", descending=True)" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "80bf3df7", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (336_776, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64str
201338null1800nullnull1937null"UA"1177null"LGA""ORD"null733180"2013-03-08T23:00:00Z"
201328null1659nullnull1822null"UA"531null"EWR""BOS"null2001659"2013-02-08T21:00:00Z"
2013813null1727nullnull1941null"EV"5892"N16561""EWR""CVG"null5691727"2013-08-13T21:00:00Z"
2013813null1225nullnull1338null"EV"5897"N15973""EWR""ORF"null2841225"2013-08-13T16:00:00Z"
2013930null840nullnull1020null"MQ"3531"N839MQ""LGA""RDU"null431840"2013-09-30T12:00:00Z"
…………………………………………………
201312917031730-2719471957-10"F9"837"N208FR""LGA""DEN"25016201730"2013-01-29T22:00:00Z"
201311119001930-3022332243-10"DL"1435"N934DL""LGA""TPA"13910101930"2013-01-12T00:00:00Z"
2013111014081440-3215491559-10"EV"5713"N825AS""LGA""IAD"522291440"2013-11-10T19:00:00Z"
20132320222055-3322402338-58"DL"1715"N612DL""LGA""MSY"16211832055"2013-02-04T01:00:00Z"
201312720402123-4340235248"B6"97"N592JB""JFK""DEN"26516262123"2013-12-08T02:00:00Z"
" - ], - "text/plain": [ - "shape: (336_776, 19)\n", - "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬──────────────────────┐\n", - "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", - "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪══════════════════════╡\n", - "│ 2013 ┆ 3 ┆ 8 ┆ null ┆ … ┆ 733 ┆ 18 ┆ 0 ┆ 2013-03-08T23:00:00Z │\n", - "│ 2013 ┆ 2 ┆ 8 ┆ null ┆ … ┆ 200 ┆ 16 ┆ 59 ┆ 2013-02-08T21:00:00Z │\n", - "│ 2013 ┆ 8 ┆ 13 ┆ null ┆ … ┆ 569 ┆ 17 ┆ 27 ┆ 2013-08-13T21:00:00Z │\n", - "│ 2013 ┆ 8 ┆ 13 ┆ null ┆ … ┆ 284 ┆ 12 ┆ 25 ┆ 2013-08-13T16:00:00Z │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 431 ┆ 8 ┆ 40 ┆ 2013-09-30T12:00:00Z │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 2013 ┆ 1 ┆ 29 ┆ 1703 ┆ … ┆ 1620 ┆ 17 ┆ 30 ┆ 2013-01-29T22:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 11 ┆ 1900 ┆ … ┆ 1010 ┆ 19 ┆ 30 ┆ 2013-01-12T00:00:00Z │\n", - "│ 2013 ┆ 11 ┆ 10 ┆ 1408 ┆ … ┆ 229 ┆ 14 ┆ 40 ┆ 2013-11-10T19:00:00Z │\n", - "│ 2013 ┆ 2 ┆ 3 ┆ 2022 ┆ … ┆ 1183 ┆ 20 ┆ 55 ┆ 2013-02-04T01:00:00Z │\n", - "│ 2013 ┆ 12 ┆ 7 ┆ 2040 ┆ … ┆ 1626 ┆ 21 ┆ 23 ┆ 2013-12-08T02:00:00Z │\n", - "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴──────────────────────┘" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights.sort([\"dep_delay\", \"arr_delay\"], descending=[True, False])" ] @@ -1073,40 +508,10 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "a939f3c2", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (3, 19)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64str
20131115251530-519341805null"MQ"4525"N719MQ""LGA""XNA"null11471530"2013-01-01T20:00:00Z"
20131117401745-521582020null"MQ"4413"N739MQ""LGA""XNA"null11471745"2013-01-01T22:00:00Z"
20131218481840823332151null"9E"3325"N920XJ""JFK""DFW"null13911840"2013-01-02T23:00:00Z"
" - ], - "text/plain": [ - "shape: (3, 19)\n", - "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────┬────────┬──────────────────────┐\n", - "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ distance ┆ hour ┆ minute ┆ time_hour │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ str │\n", - "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════╪════════╪══════════════════════╡\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 1525 ┆ … ┆ 1147 ┆ 15 ┆ 30 ┆ 2013-01-01T20:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 1740 ┆ … ┆ 1147 ┆ 17 ┆ 45 ┆ 2013-01-01T22:00:00Z │\n", - "│ 2013 ┆ 1 ┆ 2 ┆ 1848 ┆ … ┆ 1391 ┆ 18 ┆ 40 ┆ 2013-01-02T23:00:00Z │\n", - "└──────┴───────┴─────┴──────────┴───┴──────────┴──────┴────────┴──────────────────────┘" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " flights.filter((pl.col(\"dep_delay\") <= 10) & (pl.col(\"dep_delay\") >= -10))\n", @@ -1167,43 +572,10 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "86827cf9", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 8)
col0col1col2col3col4col5col6new_column0
i64i64i64i64i64i64stri32
012345"apple"5
67891011"orange"5
121314151617"pineapple"5
181920212223"mango"5
242526272829"kiwi"5
303132333435"lemon"5
" - ], - "text/plain": [ - "shape: (6, 8)\n", - "┌──────┬──────┬──────┬──────┬──────┬──────┬───────────┬─────────────┐\n", - "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 ┆ col6 ┆ new_column0 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ str ┆ i32 │\n", - "╞══════╪══════╪══════╪══════╪══════╪══════╪═══════════╪═════════════╡\n", - "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ 4 ┆ 5 ┆ apple ┆ 5 │\n", - "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ 10 ┆ 11 ┆ orange ┆ 5 │\n", - "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ 16 ┆ 17 ┆ pineapple ┆ 5 │\n", - "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ 22 ┆ 23 ┆ mango ┆ 5 │\n", - "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 ┆ kiwi ┆ 5 │\n", - "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ 34 ┆ 35 ┆ lemon ┆ 5 │\n", - "└──────┴──────┴──────┴──────┴──────┴──────┴───────────┴─────────────┘" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = df.with_columns(new_column0=pl.lit(5))\n", "df" @@ -1219,43 +591,10 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "0ab01f9d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 8)
col0col1col2col3col4col5col6new_column0
i64i64i64i64i64i64stri64
012345"apple"0
67891011"orange"1
121314151617"pineapple"2
181920212223"mango"3
242526272829"kiwi"4
303132333435"lemon"5
" - ], - "text/plain": [ - "shape: (6, 8)\n", - "┌──────┬──────┬──────┬──────┬──────┬──────┬───────────┬─────────────┐\n", - "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 ┆ col6 ┆ new_column0 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ str ┆ i64 │\n", - "╞══════╪══════╪══════╪══════╪══════╪══════╪═══════════╪═════════════╡\n", - "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ 4 ┆ 5 ┆ apple ┆ 0 │\n", - "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ 10 ┆ 11 ┆ orange ┆ 1 │\n", - "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ 16 ┆ 17 ┆ pineapple ┆ 2 │\n", - "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ 22 ┆ 23 ┆ mango ┆ 3 │\n", - "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 ┆ kiwi ┆ 4 │\n", - "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ 34 ┆ 35 ┆ lemon ┆ 5 │\n", - "└──────┴──────┴──────┴──────┴──────┴──────┴───────────┴─────────────┘" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = df.with_columns(new_column0=pl.Series([0, 1, 2, 3, 4, 5]))\n", "df" @@ -1275,43 +614,10 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "fff10e83", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 10)
col0col1col2col3col4col5col6new_column0new_column1new_column2
i64i64i64i64i64i64stri64i32i32
012345"apple"056
67891011"orange"156
121314151617"pineapple"256
181920212223"mango"356
242526272829"kiwi"456
303132333435"lemon"556
" - ], - "text/plain": [ - "shape: (6, 10)\n", - "┌──────┬──────┬──────┬──────┬───┬───────────┬─────────────┬─────────────┬─────────────┐\n", - "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ … ┆ col6 ┆ new_column0 ┆ new_column1 ┆ new_column2 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ str ┆ i64 ┆ i32 ┆ i32 │\n", - "╞══════╪══════╪══════╪══════╪═══╪═══════════╪═════════════╪═════════════╪═════════════╡\n", - "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ … ┆ apple ┆ 0 ┆ 5 ┆ 6 │\n", - "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ … ┆ orange ┆ 1 ┆ 5 ┆ 6 │\n", - "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ … ┆ pineapple ┆ 2 ┆ 5 ┆ 6 │\n", - "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ … ┆ mango ┆ 3 ┆ 5 ┆ 6 │\n", - "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ … ┆ kiwi ┆ 4 ┆ 5 ┆ 6 │\n", - "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ … ┆ lemon ┆ 5 ┆ 5 ┆ 6 │\n", - "└──────┴──────┴──────┴──────┴───┴───────────┴─────────────┴─────────────┴─────────────┘" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = df.with_columns(new_column1=pl.lit(5), new_column2=pl.lit(6))\n", "df" @@ -1327,43 +633,10 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "id": "82477100", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 10)
col0col1col2col3col4col5col6new_column0new_column1new_column2
i64i64i64i64i64i64stri64i32i64
012345"apple"050
67891011"orange"155
121314151617"pineapple"2510
181920212223"mango"3515
242526272829"kiwi"4520
303132333435"lemon"5525
" - ], - "text/plain": [ - "shape: (6, 10)\n", - "┌──────┬──────┬──────┬──────┬───┬───────────┬─────────────┬─────────────┬─────────────┐\n", - "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ … ┆ col6 ┆ new_column0 ┆ new_column1 ┆ new_column2 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ str ┆ i64 ┆ i32 ┆ i64 │\n", - "╞══════╪══════╪══════╪══════╪═══╪═══════════╪═════════════╪═════════════╪═════════════╡\n", - "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ … ┆ apple ┆ 0 ┆ 5 ┆ 0 │\n", - "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ … ┆ orange ┆ 1 ┆ 5 ┆ 5 │\n", - "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ … ┆ pineapple ┆ 2 ┆ 5 ┆ 10 │\n", - "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ … ┆ mango ┆ 3 ┆ 5 ┆ 15 │\n", - "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ … ┆ kiwi ┆ 4 ┆ 5 ┆ 20 │\n", - "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ … ┆ lemon ┆ 5 ┆ 5 ┆ 25 │\n", - "└──────┴──────┴──────┴──────┴───┴───────────┴─────────────┴─────────────┴─────────────┘" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.with_columns(new_column2=pl.col(\"col0\") - pl.col(\"new_column0\"))" ] @@ -1379,48 +652,10 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "55645bdd", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (336_776, 21)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hourgainspeed
i64i64i64i64i64i64i64i64i64stri64strstrstri64i64i64i64stri64f64
201311517515283081911"UA"1545"N14228""EWR""IAH"2271400515"2013-01-01T10:00:00Z"-9370.044053
201311533529485083020"UA"1714"N24211""LGA""IAH"2271416529"2013-01-01T10:00:00Z"-16374.273128
201311542540292385033"AA"1141"N619AA""JFK""MIA"1601089540"2013-01-01T10:00:00Z"-31408.375
201311544545-110041022-18"B6"725"N804JB""JFK""BQN"1831576545"2013-01-01T10:00:00Z"17516.721311
201311554600-6812837-25"DL"461"N668DN""LGA""ATL"11676260"2013-01-01T11:00:00Z"19394.137931
………………………………………………………
2013930null1455nullnull1634null"9E"3393null"JFK""DCA"null2131455"2013-09-30T18:00:00Z"nullnull
2013930null2200nullnull2312null"9E"3525null"LGA""SYR"null198220"2013-10-01T02:00:00Z"nullnull
2013930null1210nullnull1330null"MQ"3461"N535MQ""LGA""BNA"null7641210"2013-09-30T16:00:00Z"nullnull
2013930null1159nullnull1344null"MQ"3572"N511MQ""LGA""CLE"null4191159"2013-09-30T15:00:00Z"nullnull
2013930null840nullnull1020null"MQ"3531"N839MQ""LGA""RDU"null431840"2013-09-30T12:00:00Z"nullnull
" - ], - "text/plain": [ - "shape: (336_776, 21)\n", - "┌──────┬───────┬─────┬──────────┬───┬────────┬──────────────────────┬──────┬────────────┐\n", - "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ minute ┆ time_hour ┆ gain ┆ speed │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ str ┆ i64 ┆ f64 │\n", - "╞══════╪═══════╪═════╪══════════╪═══╪════════╪══════════════════════╪══════╪════════════╡\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 517 ┆ … ┆ 15 ┆ 2013-01-01T10:00:00Z ┆ -9 ┆ 370.044053 │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 533 ┆ … ┆ 29 ┆ 2013-01-01T10:00:00Z ┆ -16 ┆ 374.273128 │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 542 ┆ … ┆ 40 ┆ 2013-01-01T10:00:00Z ┆ -31 ┆ 408.375 │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 544 ┆ … ┆ 45 ┆ 2013-01-01T10:00:00Z ┆ 17 ┆ 516.721311 │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 554 ┆ … ┆ 0 ┆ 2013-01-01T11:00:00Z ┆ 19 ┆ 394.137931 │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 55 ┆ 2013-09-30T18:00:00Z ┆ null ┆ null │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 0 ┆ 2013-10-01T02:00:00Z ┆ null ┆ null │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 10 ┆ 2013-09-30T16:00:00Z ┆ null ┆ null │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 59 ┆ 2013-09-30T15:00:00Z ┆ null ┆ null │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ 40 ┆ 2013-09-30T12:00:00Z ┆ null ┆ null │\n", - "└──────┴───────┴─────┴──────────┴───┴────────┴──────────────────────┴──────┴────────────┘" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights.with_columns(\n", " (pl.col(\"dep_delay\") - pl.col(\"arr_delay\")).alias(\"gain\"),\n", @@ -1446,40 +681,10 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "4643b978", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6,)
col0
i64
0
6
12
18
24
30
" - ], - "text/plain": [ - "shape: (6,)\n", - "Series: 'col0' [i64]\n", - "[\n", - "\t0\n", - "\t6\n", - "\t12\n", - "\t18\n", - "\t24\n", - "\t30\n", - "]" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[\"col0\"]" ] @@ -1494,43 +699,10 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "219852d8", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 3)
col0new_column0col2
i64i64i64
002
618
12214
18320
24426
30532
" - ], - "text/plain": [ - "shape: (6, 3)\n", - "┌──────┬─────────────┬──────┐\n", - "│ col0 ┆ new_column0 ┆ col2 │\n", - "│ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 │\n", - "╞══════╪═════════════╪══════╡\n", - "│ 0 ┆ 0 ┆ 2 │\n", - "│ 6 ┆ 1 ┆ 8 │\n", - "│ 12 ┆ 2 ┆ 14 │\n", - "│ 18 ┆ 3 ┆ 20 │\n", - "│ 24 ┆ 4 ┆ 26 │\n", - "│ 30 ┆ 5 ┆ 32 │\n", - "└──────┴─────────────┴──────┘" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[[\"col0\", \"new_column0\", \"col2\"]]" ] @@ -1545,43 +717,10 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "id": "1bc0cd22", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 3)
col1col2col3
i64i64i64
002
618
12214
18320
24426
30532
" - ], - "text/plain": [ - "shape: (6, 3)\n", - "┌──────┬──────┬──────┐\n", - "│ col1 ┆ col2 ┆ col3 │\n", - "│ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 │\n", - "╞══════╪══════╪══════╡\n", - "│ 0 ┆ 0 ┆ 2 │\n", - "│ 6 ┆ 1 ┆ 8 │\n", - "│ 12 ┆ 2 ┆ 14 │\n", - "│ 18 ┆ 3 ┆ 20 │\n", - "│ 24 ┆ 4 ┆ 26 │\n", - "│ 30 ┆ 5 ┆ 32 │\n", - "└──────┴──────┴──────┘" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# selecting a single column\n", "df.select(\"col0\")\n", @@ -1603,43 +742,10 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "id": "ed447fb7", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 3)
col0new_column0col2
i64i64i64
024
6316
12428
18540
24652
30764
" - ], - "text/plain": [ - "shape: (6, 3)\n", - "┌──────┬─────────────┬──────┐\n", - "│ col0 ┆ new_column0 ┆ col2 │\n", - "│ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 │\n", - "╞══════╪═════════════╪══════╡\n", - "│ 0 ┆ 2 ┆ 4 │\n", - "│ 6 ┆ 3 ┆ 16 │\n", - "│ 12 ┆ 4 ┆ 28 │\n", - "│ 18 ┆ 5 ┆ 40 │\n", - "│ 24 ┆ 6 ┆ 52 │\n", - "│ 30 ┆ 7 ┆ 64 │\n", - "└──────┴─────────────┴──────┘" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.select(pl.col(\"col0\"), pl.col(\"new_column0\") + 2, pl.col(\"col2\") * 2)" ] @@ -1654,39 +760,10 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "id": "eabfd313", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (2, 3)
col0new_column0col2
i64i64i64
002
618
" - ], - "text/plain": [ - "shape: (2, 3)\n", - "┌──────┬─────────────┬──────┐\n", - "│ col0 ┆ new_column0 ┆ col2 │\n", - "│ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 │\n", - "╞══════╪═════════════╪══════╡\n", - "│ 0 ┆ 0 ┆ 2 │\n", - "│ 6 ┆ 1 ┆ 8 │\n", - "└──────┴─────────────┴──────┘" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.select(\"col0\", \"new_column0\", \"col2\").slice(0, 2)" ] @@ -1701,48 +778,10 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "id": "aed67406", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (336_776, 14)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delayflightair_timedistancehourminute
i64i64i64i64i64i64i64i64i64i64i64i64i64i64
20131151751528308191115452271400515
20131153352948508302017142271416529
20131154254029238503311411601089540
201311544545-110041022-187251831576545
201311554600-6812837-2546111676260
……………………………………
2013930null1455nullnull1634null3393null2131455
2013930null2200nullnull2312null3525null198220
2013930null1210nullnull1330null3461null7641210
2013930null1159nullnull1344null3572null4191159
2013930null840nullnull1020null3531null431840
" - ], - "text/plain": [ - "shape: (336_776, 14)\n", - "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────────┬──────┬────────┐\n", - "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ air_time ┆ distance ┆ hour ┆ minute │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n", - "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════════╪══════╪════════╡\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 517 ┆ … ┆ 227 ┆ 1400 ┆ 5 ┆ 15 │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 533 ┆ … ┆ 227 ┆ 1416 ┆ 5 ┆ 29 │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 542 ┆ … ┆ 160 ┆ 1089 ┆ 5 ┆ 40 │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 544 ┆ … ┆ 183 ┆ 1576 ┆ 5 ┆ 45 │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 554 ┆ … ┆ 116 ┆ 762 ┆ 6 ┆ 0 │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 213 ┆ 14 ┆ 55 │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 198 ┆ 22 ┆ 0 │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 764 ┆ 12 ┆ 10 │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 419 ┆ 11 ┆ 59 │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 431 ┆ 8 ┆ 40 │\n", - "└──────┴───────┴─────┴──────────┴───┴──────────┴──────────┴──────┴────────┘" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights.select(pl.col(pl.Int64))" ] @@ -1757,48 +796,10 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "id": "62f578d1", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (336_776, 14)
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delayflightair_timedistancehourminute
i64i64i64i64i64i64i64i64i64i64i64i64i64i64
20131151751528308191115452271400515
20131153352948508302017142271416529
20131154254029238503311411601089540
201311544545-110041022-187251831576545
201311554600-6812837-2546111676260
……………………………………
2013930null1455nullnull1634null3393null2131455
2013930null2200nullnull2312null3525null198220
2013930null1210nullnull1330null3461null7641210
2013930null1159nullnull1344null3572null4191159
2013930null840nullnull1020null3531null431840
" - ], - "text/plain": [ - "shape: (336_776, 14)\n", - "┌──────┬───────┬─────┬──────────┬───┬──────────┬──────────┬──────┬────────┐\n", - "│ year ┆ month ┆ day ┆ dep_time ┆ … ┆ air_time ┆ distance ┆ hour ┆ minute │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n", - "╞══════╪═══════╪═════╪══════════╪═══╪══════════╪══════════╪══════╪════════╡\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 517 ┆ … ┆ 227 ┆ 1400 ┆ 5 ┆ 15 │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 533 ┆ … ┆ 227 ┆ 1416 ┆ 5 ┆ 29 │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 542 ┆ … ┆ 160 ┆ 1089 ┆ 5 ┆ 40 │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 544 ┆ … ┆ 183 ┆ 1576 ┆ 5 ┆ 45 │\n", - "│ 2013 ┆ 1 ┆ 1 ┆ 554 ┆ … ┆ 116 ┆ 762 ┆ 6 ┆ 0 │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 213 ┆ 14 ┆ 55 │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 198 ┆ 22 ┆ 0 │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 764 ┆ 12 ┆ 10 │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 419 ┆ 11 ┆ 59 │\n", - "│ 2013 ┆ 9 ┆ 30 ┆ null ┆ … ┆ null ┆ 431 ┆ 8 ┆ 40 │\n", - "└──────┴───────┴─────┴──────────┴───┴──────────┴──────────┴──────┴────────┘" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import polars.selectors as S\n", "\n", @@ -1811,48 +812,10 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "id": "d4e486db", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (336_776, 2)
arr_timearr_delay
i64i64
83011
85020
92333
1004-18
812-25
……
nullnull
nullnull
nullnull
nullnull
nullnull
" - ], - "text/plain": [ - "shape: (336_776, 2)\n", - "┌──────────┬───────────┐\n", - "│ arr_time ┆ arr_delay │\n", - "│ --- ┆ --- │\n", - "│ i64 ┆ i64 │\n", - "╞══════════╪═══════════╡\n", - "│ 830 ┆ 11 │\n", - "│ 850 ┆ 20 │\n", - "│ 923 ┆ 33 │\n", - "│ 1004 ┆ -18 │\n", - "│ 812 ┆ -25 │\n", - "│ … ┆ … │\n", - "│ null ┆ null │\n", - "│ null ┆ null │\n", - "│ null ┆ null │\n", - "│ null ┆ null │\n", - "│ null ┆ null │\n", - "└──────────┴───────────┘" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Select columns that contain \"delay\" in their name\n", "flights.select(S.contains(\"delay\"))\n", @@ -1881,43 +844,10 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "id": "5e5c0dd0", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 10)
col0col1col2lettersnamescol5fruitnew_column0new_column1new_column2
i64i64i64i64i64i64stri64i32i32
012345"apple"056
67891011"orange"156
121314151617"pineapple"256
181920212223"mango"356
242526272829"kiwi"456
303132333435"lemon"556
" - ], - "text/plain": [ - "shape: (6, 10)\n", - "┌──────┬──────┬──────┬─────────┬───┬───────────┬─────────────┬─────────────┬─────────────┐\n", - "│ col0 ┆ col1 ┆ col2 ┆ letters ┆ … ┆ fruit ┆ new_column0 ┆ new_column1 ┆ new_column2 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ str ┆ i64 ┆ i32 ┆ i32 │\n", - "╞══════╪══════╪══════╪═════════╪═══╪═══════════╪═════════════╪═════════════╪═════════════╡\n", - "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ … ┆ apple ┆ 0 ┆ 5 ┆ 6 │\n", - "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ … ┆ orange ┆ 1 ┆ 5 ┆ 6 │\n", - "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ … ┆ pineapple ┆ 2 ┆ 5 ┆ 6 │\n", - "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ … ┆ mango ┆ 3 ┆ 5 ┆ 6 │\n", - "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ … ┆ kiwi ┆ 4 ┆ 5 ┆ 6 │\n", - "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ … ┆ lemon ┆ 5 ┆ 5 ┆ 6 │\n", - "└──────┴──────┴──────┴─────────┴───┴───────────┴─────────────┴─────────────┴─────────────┘" - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.rename({\"col3\": \"letters\", \"col4\": \"names\", \"col6\": \"fruit\"})" ] @@ -1932,43 +862,10 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "id": "482d301f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 10)
COL0COL1COL2COL3COL4COL5COL6NEW_COLUMN0NEW_COLUMN1NEW_COLUMN2
i64i64i64i64i64i64stri64i32i32
012345"apple"056
67891011"orange"156
121314151617"pineapple"256
181920212223"mango"356
242526272829"kiwi"456
303132333435"lemon"556
" - ], - "text/plain": [ - "shape: (6, 10)\n", - "┌──────┬──────┬──────┬──────┬───┬───────────┬─────────────┬─────────────┬─────────────┐\n", - "│ COL0 ┆ COL1 ┆ COL2 ┆ COL3 ┆ … ┆ COL6 ┆ NEW_COLUMN0 ┆ NEW_COLUMN1 ┆ NEW_COLUMN2 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ str ┆ i64 ┆ i32 ┆ i32 │\n", - "╞══════╪══════╪══════╪══════╪═══╪═══════════╪═════════════╪═════════════╪═════════════╡\n", - "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ … ┆ apple ┆ 0 ┆ 5 ┆ 6 │\n", - "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ … ┆ orange ┆ 1 ┆ 5 ┆ 6 │\n", - "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ … ┆ pineapple ┆ 2 ┆ 5 ┆ 6 │\n", - "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ … ┆ mango ┆ 3 ┆ 5 ┆ 6 │\n", - "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ … ┆ kiwi ┆ 4 ┆ 5 ┆ 6 │\n", - "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ … ┆ lemon ┆ 5 ┆ 5 ┆ 6 │\n", - "└──────┴──────┴──────┴──────┴───┴───────────┴─────────────┴─────────────┴─────────────┘" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.rename(lambda column_name: column_name.upper())" ] @@ -2005,43 +902,10 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "id": "b7c0d519", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 6)
col0col1col2col3col4col5
i64i64i64i64i64i64
012345
67891011
121314151617
181920212223
242526272829
303132333435
" - ], - "text/plain": [ - "shape: (6, 6)\n", - "┌──────┬──────┬──────┬──────┬──────┬──────┐\n", - "│ col0 ┆ col1 ┆ col2 ┆ col3 ┆ col4 ┆ col5 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n", - "╞══════╪══════╪══════╪══════╪══════╪══════╡\n", - "│ 0 ┆ 1 ┆ 2 ┆ 3 ┆ 4 ┆ 5 │\n", - "│ 6 ┆ 7 ┆ 8 ┆ 9 ┆ 10 ┆ 11 │\n", - "│ 12 ┆ 13 ┆ 14 ┆ 15 ┆ 16 ┆ 17 │\n", - "│ 18 ┆ 19 ┆ 20 ┆ 21 ┆ 22 ┆ 23 │\n", - "│ 24 ┆ 25 ┆ 26 ┆ 27 ┆ 28 ┆ 29 │\n", - "│ 30 ┆ 31 ┆ 32 ┆ 33 ┆ 34 ┆ 35 │\n", - "└──────┴──────┴──────┴──────┴──────┴──────┘" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = pl.DataFrame(\n", " data=np.reshape(range(36), (6, 6)), schema=[\"col\" + str(i) for i in range(6)]\n", @@ -2051,43 +915,10 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "id": "3c2029cc", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 6)
col5col3col1col4col2col0
i64i64i64i64i64i64
531420
11971086
171513161412
232119222018
292725282624
353331343230
" - ], - "text/plain": [ - "shape: (6, 6)\n", - "┌──────┬──────┬──────┬──────┬──────┬──────┐\n", - "│ col5 ┆ col3 ┆ col1 ┆ col4 ┆ col2 ┆ col0 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n", - "╞══════╪══════╪══════╪══════╪══════╪══════╡\n", - "│ 5 ┆ 3 ┆ 1 ┆ 4 ┆ 2 ┆ 0 │\n", - "│ 11 ┆ 9 ┆ 7 ┆ 10 ┆ 8 ┆ 6 │\n", - "│ 17 ┆ 15 ┆ 13 ┆ 16 ┆ 14 ┆ 12 │\n", - "│ 23 ┆ 21 ┆ 19 ┆ 22 ┆ 20 ┆ 18 │\n", - "│ 29 ┆ 27 ┆ 25 ┆ 28 ┆ 26 ┆ 24 │\n", - "│ 35 ┆ 33 ┆ 31 ┆ 34 ┆ 32 ┆ 30 │\n", - "└──────┴──────┴──────┴──────┴──────┴──────┘" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = df.select([\"col5\", \"col3\", \"col1\", \"col4\", \"col2\", \"col0\"])\n", "df" @@ -2103,43 +934,10 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "id": "e7ab5f64", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 6)
col5col4col3col2col1col0
i64i64i64i64i64i64
543210
11109876
171615141312
232221201918
292827262524
353433323130
" - ], - "text/plain": [ - "shape: (6, 6)\n", - "┌──────┬──────┬──────┬──────┬──────┬──────┐\n", - "│ col5 ┆ col4 ┆ col3 ┆ col2 ┆ col1 ┆ col0 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n", - "╞══════╪══════╪══════╪══════╪══════╪══════╡\n", - "│ 5 ┆ 4 ┆ 3 ┆ 2 ┆ 1 ┆ 0 │\n", - "│ 11 ┆ 10 ┆ 9 ┆ 8 ┆ 7 ┆ 6 │\n", - "│ 17 ┆ 16 ┆ 15 ┆ 14 ┆ 13 ┆ 12 │\n", - "│ 23 ┆ 22 ┆ 21 ┆ 20 ┆ 19 ┆ 18 │\n", - "│ 29 ┆ 28 ┆ 27 ┆ 26 ┆ 25 ┆ 24 │\n", - "│ 35 ┆ 34 ┆ 33 ┆ 32 ┆ 31 ┆ 30 │\n", - "└──────┴──────┴──────┴──────┴──────┴──────┘" - ] - }, - "execution_count": 40, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Alphabetical order\n", "df.select(sorted(df.columns))\n", @@ -2184,48 +982,10 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "id": "62e540c0", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (12, 2)
monthdep_delay
i64f64
721.727787
620.846332
512.986859
1216.576688
210.816843
……
812.61104
110.036665
313.227076
96.722476
106.243988
" - ], - "text/plain": [ - "shape: (12, 2)\n", - "┌───────┬───────────┐\n", - "│ month ┆ dep_delay │\n", - "│ --- ┆ --- │\n", - "│ i64 ┆ f64 │\n", - "╞═══════╪═══════════╡\n", - "│ 7 ┆ 21.727787 │\n", - "│ 6 ┆ 20.846332 │\n", - "│ 5 ┆ 12.986859 │\n", - "│ 12 ┆ 16.576688 │\n", - "│ 2 ┆ 10.816843 │\n", - "│ … ┆ … │\n", - "│ 8 ┆ 12.61104 │\n", - "│ 1 ┆ 10.036665 │\n", - "│ 3 ┆ 13.227076 │\n", - "│ 9 ┆ 6.722476 │\n", - "│ 10 ┆ 6.243988 │\n", - "└───────┴───────────┘" - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights.group_by(\"month\").agg(pl.col(\"dep_delay\").mean())" ] @@ -2263,48 +1023,10 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "id": "af588177", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (12, 3)
monthmean_delaycount_flights
i64f64u32
313.22707627973
1216.57668827110
620.84633227234
96.72247627122
110.03666526483
………
721.72778728485
210.81684323690
812.6110428841
512.98685928233
115.43536227035
" - ], - "text/plain": [ - "shape: (12, 3)\n", - "┌───────┬────────────┬───────────────┐\n", - "│ month ┆ mean_delay ┆ count_flights │\n", - "│ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ f64 ┆ u32 │\n", - "╞═══════╪════════════╪═══════════════╡\n", - "│ 3 ┆ 13.227076 ┆ 27973 │\n", - "│ 12 ┆ 16.576688 ┆ 27110 │\n", - "│ 6 ┆ 20.846332 ┆ 27234 │\n", - "│ 9 ┆ 6.722476 ┆ 27122 │\n", - "│ 1 ┆ 10.036665 ┆ 26483 │\n", - "│ … ┆ … ┆ … │\n", - "│ 7 ┆ 21.727787 ┆ 28485 │\n", - "│ 2 ┆ 10.816843 ┆ 23690 │\n", - "│ 8 ┆ 12.61104 ┆ 28841 │\n", - "│ 5 ┆ 12.986859 ┆ 28233 │\n", - "│ 11 ┆ 5.435362 ┆ 27035 │\n", - "└───────┴────────────┴───────────────┘" - ] - }, - "execution_count": 42, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Multiple aggregations using polars' syntactic sugar (shorthand) for mean and count\n", "flights.group_by(\"month\").agg(\n", @@ -2333,48 +1055,10 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "id": "b0e56ff1", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (12, 4)
monthyearmean_delaycount_flights
i64i64f64u32
2201310.81684323690
6201320.84633227234
7201321.72778728485
1020136.24398828653
1201310.03666526483
…………
920136.72247627122
3201313.22707627973
1120135.43536227035
12201316.57668827110
4201313.93803827662
" - ], - "text/plain": [ - "shape: (12, 4)\n", - "┌───────┬──────┬────────────┬───────────────┐\n", - "│ month ┆ year ┆ mean_delay ┆ count_flights │\n", - "│ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ i64 ┆ f64 ┆ u32 │\n", - "╞═══════╪══════╪════════════╪═══════════════╡\n", - "│ 2 ┆ 2013 ┆ 10.816843 ┆ 23690 │\n", - "│ 6 ┆ 2013 ┆ 20.846332 ┆ 27234 │\n", - "│ 7 ┆ 2013 ┆ 21.727787 ┆ 28485 │\n", - "│ 10 ┆ 2013 ┆ 6.243988 ┆ 28653 │\n", - "│ 1 ┆ 2013 ┆ 10.036665 ┆ 26483 │\n", - "│ … ┆ … ┆ … ┆ … │\n", - "│ 9 ┆ 2013 ┆ 6.722476 ┆ 27122 │\n", - "│ 3 ┆ 2013 ┆ 13.227076 ┆ 27973 │\n", - "│ 11 ┆ 2013 ┆ 5.435362 ┆ 27035 │\n", - "│ 12 ┆ 2013 ┆ 16.576688 ┆ 27110 │\n", - "│ 4 ┆ 2013 ┆ 13.938038 ┆ 27662 │\n", - "└───────┴──────┴────────────┴───────────────┘" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "month_year_delay = flights.group_by(\"month\", \"year\").agg(\n", " mean_delay=pl.mean(\"dep_delay\"),\n", diff --git a/data-visualise.ipynb b/data-visualise.ipynb index 0e2feeb..8f3d721 100644 --- a/data-visualise.ipynb +++ b/data-visualise.ipynb @@ -62,51 +62,10 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "a86fb211", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "import polars as pl\n", "from lets_plot import *\n", @@ -163,49 +122,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "0cf986aa", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (344, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64f64f64stri64
"Adelie""Torgersen"39.118.7181.03750.0"male"2007
"Adelie""Torgersen"39.517.4186.03800.0"female"2007
"Adelie""Torgersen"40.318.0195.03250.0"female"2007
"Adelie""Torgersen"nullnullnullnullnull2007
"Adelie""Torgersen"36.719.3193.03450.0"female"2007
……………………
"Chinstrap""Dream"55.819.8207.04000.0"male"2009
"Chinstrap""Dream"43.518.1202.03400.0"female"2009
"Chinstrap""Dream"49.618.2193.03775.0"male"2009
"Chinstrap""Dream"50.819.0210.04100.0"male"2009
"Chinstrap""Dream"50.218.7198.03775.0"female"2009
" - ], - "text/plain": [ - "shape: (344, 8)\n", - "┌───────────┬───────────┬──────────────┬──────────────┬──────────────┬─────────────┬────────┬──────┐\n", - "│ species ┆ island ┆ bill_length_ ┆ bill_depth_m ┆ flipper_leng ┆ body_mass_g ┆ sex ┆ year │\n", - "│ --- ┆ --- ┆ mm ┆ m ┆ th_mm ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ --- ┆ --- ┆ --- ┆ f64 ┆ str ┆ i64 │\n", - "│ ┆ ┆ f64 ┆ f64 ┆ f64 ┆ ┆ ┆ │\n", - "╞═══════════╪═══════════╪══════════════╪══════════════╪══════════════╪═════════════╪════════╪══════╡\n", - "│ Adelie ┆ Torgersen ┆ 39.1 ┆ 18.7 ┆ 181.0 ┆ 3750.0 ┆ male ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 39.5 ┆ 17.4 ┆ 186.0 ┆ 3800.0 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 40.3 ┆ 18.0 ┆ 195.0 ┆ 3250.0 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ null ┆ null ┆ null ┆ null ┆ null ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 36.7 ┆ 19.3 ┆ 193.0 ┆ 3450.0 ┆ female ┆ 2007 │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ Chinstrap ┆ Dream ┆ 55.8 ┆ 19.8 ┆ 207.0 ┆ 4000.0 ┆ male ┆ 2009 │\n", - "│ Chinstrap ┆ Dream ┆ 43.5 ┆ 18.1 ┆ 202.0 ┆ 3400.0 ┆ female ┆ 2009 │\n", - "│ Chinstrap ┆ Dream ┆ 49.6 ┆ 18.2 ┆ 193.0 ┆ 3775.0 ┆ male ┆ 2009 │\n", - "│ Chinstrap ┆ Dream ┆ 50.8 ┆ 19.0 ┆ 210.0 ┆ 4100.0 ┆ male ┆ 2009 │\n", - "│ Chinstrap ┆ Dream ┆ 50.2 ┆ 18.7 ┆ 198.0 ┆ 3775.0 ┆ female ┆ 2009 │\n", - "└───────────┴───────────┴──────────────┴──────────────┴──────────────┴─────────────┴────────┴──────┘" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "penguins = pl.from_pandas(load_penguins())\n", "penguins" @@ -221,43 +141,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "23c75ba7", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (5, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64f64f64stri64
"Adelie""Torgersen"39.118.7181.03750.0"male"2007
"Adelie""Torgersen"39.517.4186.03800.0"female"2007
"Adelie""Torgersen"40.318.0195.03250.0"female"2007
"Adelie""Torgersen"nullnullnullnullnull2007
"Adelie""Torgersen"36.719.3193.03450.0"female"2007
" - ], - "text/plain": [ - "shape: (5, 8)\n", - "┌─────────┬───────────┬───────────────┬───────────────┬──────────────┬─────────────┬────────┬──────┐\n", - "│ species ┆ island ┆ bill_length_m ┆ bill_depth_mm ┆ flipper_leng ┆ body_mass_g ┆ sex ┆ year │\n", - "│ --- ┆ --- ┆ m ┆ --- ┆ th_mm ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ --- ┆ f64 ┆ --- ┆ f64 ┆ str ┆ i64 │\n", - "│ ┆ ┆ f64 ┆ ┆ f64 ┆ ┆ ┆ │\n", - "╞═════════╪═══════════╪═══════════════╪═══════════════╪══════════════╪═════════════╪════════╪══════╡\n", - "│ Adelie ┆ Torgersen ┆ 39.1 ┆ 18.7 ┆ 181.0 ┆ 3750.0 ┆ male ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 39.5 ┆ 17.4 ┆ 186.0 ┆ 3800.0 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 40.3 ┆ 18.0 ┆ 195.0 ┆ 3250.0 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ null ┆ null ┆ null ┆ null ┆ null ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 36.7 ┆ 19.3 ┆ 193.0 ┆ 3450.0 ┆ female ┆ 2007 │\n", - "└─────────┴───────────┴───────────────┴───────────────┴──────────────┴─────────────┴────────┴──────┘" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "penguins.head()" ] @@ -290,158 +177,14 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "574fe39f", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(penguins, aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -504,120 +247,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "15c3848b", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(data=penguins, mapping=aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -658,122 +291,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "6b0e1c38", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(\n", @@ -804,136 +325,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "943efd36", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(\n", @@ -961,135 +356,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "9e12b3bf", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(data=penguins, mapping=aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -1112,136 +382,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "17d5803b", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(data=penguins, mapping=aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -1262,154 +406,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "b9b98ec4", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(data=penguins, mapping=aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -1470,138 +470,14 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "7c76be4b", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(data=penguins, mapping=aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -1714,119 +590,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "21b45061", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(penguins, aes(x=\"species\")) + geom_bar())" ] @@ -1843,43 +610,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "4e046bb2", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (5, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
catstrf64f64f64f64stri64
"Adelie""Torgersen"39.118.7181.03750.0"male"2007
"Adelie""Torgersen"39.517.4186.03800.0"female"2007
"Adelie""Torgersen"40.318.0195.03250.0"female"2007
"Adelie""Torgersen"nullnullnullnullnull2007
"Adelie""Torgersen"36.719.3193.03450.0"female"2007
" - ], - "text/plain": [ - "shape: (5, 8)\n", - "┌─────────┬───────────┬───────────────┬───────────────┬──────────────┬─────────────┬────────┬──────┐\n", - "│ species ┆ island ┆ bill_length_m ┆ bill_depth_mm ┆ flipper_leng ┆ body_mass_g ┆ sex ┆ year │\n", - "│ --- ┆ --- ┆ m ┆ --- ┆ th_mm ┆ --- ┆ --- ┆ --- │\n", - "│ cat ┆ str ┆ --- ┆ f64 ┆ --- ┆ f64 ┆ str ┆ i64 │\n", - "│ ┆ ┆ f64 ┆ ┆ f64 ┆ ┆ ┆ │\n", - "╞═════════╪═══════════╪═══════════════╪═══════════════╪══════════════╪═════════════╪════════╪══════╡\n", - "│ Adelie ┆ Torgersen ┆ 39.1 ┆ 18.7 ┆ 181.0 ┆ 3750.0 ┆ male ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 39.5 ┆ 17.4 ┆ 186.0 ┆ 3800.0 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 40.3 ┆ 18.0 ┆ 195.0 ┆ 3250.0 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ null ┆ null ┆ null ┆ null ┆ null ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 36.7 ┆ 19.3 ┆ 193.0 ┆ 3450.0 ┆ female ┆ 2007 │\n", - "└─────────┴───────────┴───────────────┴───────────────┴──────────────┴─────────────┴────────┴──────┘" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "penguins = penguins.cast({\"species\": pl.Categorical})\n", "penguins.head()" @@ -1907,120 +641,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "93675336", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(penguins, aes(x=\"body_mass_g\")) + geom_histogram(binwidth=200))" ] @@ -2056,120 +680,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "6a58021f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(penguins, aes(x=\"body_mass_g\")) + geom_density())" ] @@ -2238,141 +752,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "a636947a", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(penguins, aes(x=\"species\", y=\"body_mass_g\")) + geom_boxplot())" ] @@ -2387,123 +770,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "9b85a2df", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(penguins, aes(x=\"body_mass_g\", color=\"species\")) + geom_density(size=2))" ] @@ -2522,124 +792,10 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "353189e5", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(penguins, aes(x=\"body_mass_g\", color=\"species\", fill=\"species\"))\n", @@ -2677,121 +833,10 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "e091e211", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(penguins, aes(x=\"island\", fill=\"species\")) + geom_bar())" ] @@ -2808,122 +853,10 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "7df8fb7a", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(penguins, aes(x=\"island\", fill=\"species\")) + geom_bar(position=\"fill\"))" ] @@ -2949,120 +882,10 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "5066527d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(penguins, aes(x=\"flipper_length_mm\", y=\"body_mass_g\")) + geom_point())" ] @@ -3081,124 +904,10 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "8ca23d34", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(penguins, aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -3222,130 +931,10 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "00dd36e3", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(penguins, aes(x=\"flipper_length_mm\", y=\"body_mass_g\"))\n", @@ -3415,21 +1004,10 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "3410634b", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'/Users/omagic/Documents/GitHub/python4DSpolars/lets-plot-images/penguin-plot.svg'" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "plotted_data = (\n", " ggplot(penguins, aes(x=\"flipper_length_mm\", y=\"body_mass_g\")) + geom_point()\n", @@ -3451,7 +1029,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "852afe51", "metadata": { "tags": [ diff --git a/databases.ipynb b/databases.ipynb index 3c786b9..b183959 100644 --- a/databases.ipynb +++ b/databases.ipynb @@ -22,7 +22,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "9e54fbf6", "metadata": {}, "outputs": [], @@ -92,30 +92,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "970d2c19", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[(1, 'AC/DC'),\n", - " (2, 'Accept'),\n", - " (3, 'Aerosmith'),\n", - " (4, 'Alanis Morissette'),\n", - " (5, 'Alice In Chains'),\n", - " (6, 'Antônio Carlos Jobim'),\n", - " (7, 'Apocalyptica'),\n", - " (8, 'Audioslave'),\n", - " (9, 'BackBeat'),\n", - " (10, 'Billy Cobham')]" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import sqlite3\n", "\n", @@ -136,47 +116,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "c5871b6e", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (10, 2)
column_0column_1
i64str
1"AC/DC"
2"Accept"
3"Aerosmith"
4"Alanis Morissette"
5"Alice In Chains"
6"Antônio Carlos Jobim"
7"Apocalyptica"
8"Audioslave"
9"BackBeat"
10"Billy Cobham"
" - ], - "text/plain": [ - "shape: (10, 2)\n", - "┌──────────┬──────────────────────┐\n", - "│ column_0 ┆ column_1 │\n", - "│ --- ┆ --- │\n", - "│ i64 ┆ str │\n", - "╞══════════╪══════════════════════╡\n", - "│ 1 ┆ AC/DC │\n", - "│ 2 ┆ Accept │\n", - "│ 3 ┆ Aerosmith │\n", - "│ 4 ┆ Alanis Morissette │\n", - "│ 5 ┆ Alice In Chains │\n", - "│ 6 ┆ Antônio Carlos Jobim │\n", - "│ 7 ┆ Apocalyptica │\n", - "│ 8 ┆ Audioslave │\n", - "│ 9 ┆ BackBeat │\n", - "│ 10 ┆ Billy Cobham │\n", - "└──────────┴──────────────────────┘" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import polars as pl\n", "\n", @@ -195,21 +138,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "62791eab", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['ArtistId', 'Name']" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "[i[0] for i in cursor.description]" ] @@ -226,7 +158,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "0074d4d3", "metadata": {}, "outputs": [], @@ -249,7 +181,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "8020a73a", "metadata": {}, "outputs": [], @@ -270,21 +202,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "8a8354b2", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[('US', 1.0, 3), ('UK', 0.6, 2), ('France', 0.8, 1)]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "con_new.execute(\"SELECT * FROM test\").fetchall()" ] @@ -299,7 +220,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "927e804d", "metadata": { "tags": [ @@ -357,30 +278,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "be55f957", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[('Bodies', 180035),\n", - " ('Vivo Isolado Do Mundo', 180035),\n", - " ('Elvis Ate America', 180166),\n", - " ('Remote Control', 180297),\n", - " ('Promises', 180401),\n", - " ('Emergency', 180427),\n", - " ('À Vontade (Live Mix)', 180636),\n", - " ('Hyperconectividade', 180636),\n", - " ('On Fire', 180636),\n", - " ('Fascinação', 180793)]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "sql_query = \"SELECT name, milliseconds FROM track WHERE milliseconds > 1e3*3*60 ORDER BY milliseconds ASC LIMIT 10;\"\n", "cursor = con.execute(sql_query)\n", @@ -398,25 +299,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "3f894066", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[(340, 0.863),\n", - " (345, 1.11065),\n", - " (318, 1.6882166666666667),\n", - " (314, 1.69135),\n", - " (328, 1.8377666666666668)]" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "sql_groupby = \"SELECT albumid, AVG(milliseconds)/1e3/60 FROM track GROUP BY albumid ORDER BY AVG(milliseconds) ASC LIMIT 5;\"\n", "cursor = con.execute(sql_groupby)\n", @@ -438,25 +324,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "9824b70a", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[(340, 0.863, \"Liszt - 12 Études D'Execution Transcendante\"),\n", - " (345, 1.11065, \"Monteverdi: L'Orfeo\"),\n", - " (318, 1.6882166666666667, 'SCRIABIN: Vers la flamme'),\n", - " (314, 1.69135, 'English Renaissance'),\n", - " (328, 1.8377666666666668, 'Charpentier: Divertissements, Airs & Concerts')]" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "sql_join = \"SELECT track.albumid, AVG(milliseconds)/1e3/60, album.title FROM track INNER JOIN album ON (track.albumid = album.albumid) GROUP BY album.albumid ORDER BY AVG(milliseconds) ASC LIMIT 5;\"\n", "cursor = con.execute(sql_join)\n", @@ -540,7 +411,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "f5c83909", "metadata": {}, "outputs": [], @@ -577,45 +448,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "fc89c429", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
-       "┃ TrackId ┃ Name                                    ┃ AlbumId ┃ MediaTypeId ┃ GenreId ┃ Composer                                                               ┃ Milliseconds ┃ Bytes    ┃ UnitPrice       ┃\n",
-       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
-       "│ !int64  │ !string                                 │ int64   │ !int64      │ int64   │ string                                                                 │ !int64       │ int64    │ !decimal(10, 2) │\n",
-       "├─────────┼─────────────────────────────────────────┼─────────┼─────────────┼─────────┼────────────────────────────────────────────────────────────────────────┼──────────────┼──────────┼─────────────────┤\n",
-       "│       1 │ For Those About To Rock (We Salute You) │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson                              │       343719 │ 11170334 │            0.99 │\n",
-       "│       2 │ Balls to the Wall                       │       2 │           2 │       1 │ NULL                                                                   │       342562 │  5510424 │            0.99 │\n",
-       "│       3 │ Fast As a Shark                         │       3 │           2 │       1 │ F. Baltes, S. Kaufman, U. Dirkscneider & W. Hoffman                    │       230619 │  3990994 │            0.99 │\n",
-       "│       4 │ Restless and Wild                       │       3 │           2 │       1 │ F. Baltes, R.A. Smith-Diesel, S. Kaufman, U. Dirkscneider & W. Hoffman │       252051 │  4331779 │            0.99 │\n",
-       "│       5 │ Princess of the Dawn                    │       3 │           2 │       1 │ Deaffy & R.A. Smith-Diesel                                             │       375418 │  6290521 │            0.99 │\n",
-       "└─────────┴─────────────────────────────────────────┴─────────┴─────────────┴─────────┴────────────────────────────────────────────────────────────────────────┴──────────────┴──────────┴─────────────────┘\n",
-       "
\n" - ], - "text/plain": [ - "┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n", - "┃\u001b[1m \u001b[0m\u001b[1mTrackId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mName\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mAlbumId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMediaTypeId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mGenreId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mComposer\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMilliseconds\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mBytes\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mUnitPrice\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\n", - "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n", - "│ \u001b[2m!int64\u001b[0m │ \u001b[2m!string\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2m!int64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2m!int64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2m!decimal(10, 2)\u001b[0m │\n", - "├─────────┼─────────────────────────────────────────┼─────────┼─────────────┼─────────┼────────────────────────────────────────────────────────────────────────┼──────────────┼──────────┼─────────────────┤\n", - "│ \u001b[1;36m1\u001b[0m │ \u001b[32mFor Those About To Rock (We Salute You)\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson \u001b[0m │ \u001b[1;36m343719\u001b[0m │ \u001b[1;36m11170334\u001b[0m │ \u001b[1;36m0.99\u001b[0m │\n", - "│ \u001b[1;36m2\u001b[0m │ \u001b[32mBalls to the Wall \u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[1;36m342562\u001b[0m │ \u001b[1;36m5510424\u001b[0m │ \u001b[1;36m0.99\u001b[0m │\n", - "│ \u001b[1;36m3\u001b[0m │ \u001b[32mFast As a Shark \u001b[0m │ \u001b[1;36m3\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mF. Baltes, S. Kaufman, U. Dirkscneider & W. Hoffman \u001b[0m │ \u001b[1;36m230619\u001b[0m │ \u001b[1;36m3990994\u001b[0m │ \u001b[1;36m0.99\u001b[0m │\n", - "│ \u001b[1;36m4\u001b[0m │ \u001b[32mRestless and Wild \u001b[0m │ \u001b[1;36m3\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mF. Baltes, R.A. Smith-Diesel, S. Kaufman, U. Dirkscneider & W. Hoffman\u001b[0m │ \u001b[1;36m252051\u001b[0m │ \u001b[1;36m4331779\u001b[0m │ \u001b[1;36m0.99\u001b[0m │\n", - "│ \u001b[1;36m5\u001b[0m │ \u001b[32mPrincess of the Dawn \u001b[0m │ \u001b[1;36m3\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mDeaffy & R.A. Smith-Diesel \u001b[0m │ \u001b[1;36m375418\u001b[0m │ \u001b[1;36m6290521\u001b[0m │ \u001b[1;36m0.99\u001b[0m │\n", - "└─────────┴─────────────────────────────────────────┴─────────┴─────────────┴─────────┴────────────────────────────────────────────────────────────────────────┴──────────────┴──────────┴─────────────────┘" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import ibis\n", "\n", @@ -635,45 +471,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "5e5a482b", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
-       "┃ TrackId ┃ Name                                                     ┃ AlbumId ┃ MediaTypeId ┃ GenreId ┃ Composer           ┃ Milliseconds ┃ Bytes   ┃ UnitPrice       ┃ mean_mins_track ┃\n",
-       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
-       "│ !int64  │ !string                                                  │ int64   │ !int64      │ int64   │ string             │ !int64       │ int64   │ !decimal(10, 2) │ float64         │\n",
-       "├─────────┼──────────────────────────────────────────────────────────┼─────────┼─────────────┼─────────┼────────────────────┼──────────────┼─────────┼─────────────────┼─────────────────┤\n",
-       "│    3496 │ Étude 1, In C Major - Preludio (Presto) - Liszt          │     340 │           4 │      24 │ NULL               │        51780 │ 2229617 │            0.99 │        0.863000 │\n",
-       "│    3501 │ L'orfeo, Act 3, Sinfonia (Orchestra)                     │     345 │           2 │      24 │ Claudio Monteverdi │        66639 │ 1189062 │            0.99 │        1.110650 │\n",
-       "│    3452 │ SCRIABIN: Prelude in B Major, Op. 11, No. 11             │     318 │           4 │      24 │ NULL               │       101293 │ 3819535 │            0.99 │        1.688217 │\n",
-       "│    3448 │ Lamentations of Jeremiah, First Set \\ Incipit Lamentatio │     314 │           2 │      24 │ Thomas Tallis      │        69194 │ 1208080 │            0.99 │        1.691350 │\n",
-       "│    3492 │ Sing Joyfully                                            │     314 │           2 │      24 │ William Byrd       │       133768 │ 2256484 │            0.99 │        1.691350 │\n",
-       "└─────────┴──────────────────────────────────────────────────────────┴─────────┴─────────────┴─────────┴────────────────────┴──────────────┴─────────┴─────────────────┴─────────────────┘\n",
-       "
\n" - ], - "text/plain": [ - "┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n", - "┃\u001b[1m \u001b[0m\u001b[1mTrackId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mName\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mAlbumId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMediaTypeId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mGenreId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mComposer\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMilliseconds\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mBytes\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mUnitPrice\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mmean_mins_track\u001b[0m\u001b[1m \u001b[0m┃\n", - "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n", - "│ \u001b[2m!int64\u001b[0m │ \u001b[2m!string\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2m!int64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2m!int64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2m!decimal(10, 2)\u001b[0m │ \u001b[2mfloat64\u001b[0m │\n", - "├─────────┼──────────────────────────────────────────────────────────┼─────────┼─────────────┼─────────┼────────────────────┼──────────────┼─────────┼─────────────────┼─────────────────┤\n", - "│ \u001b[1;36m3496\u001b[0m │ \u001b[32mÉtude 1, In C Major - Preludio (Presto) - Liszt \u001b[0m │ \u001b[1;36m340\u001b[0m │ \u001b[1;36m4\u001b[0m │ \u001b[1;36m24\u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[1;36m51780\u001b[0m │ \u001b[1;36m2229617\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m0.863000\u001b[0m │\n", - "│ \u001b[1;36m3501\u001b[0m │ \u001b[32mL'orfeo, Act 3, Sinfonia (Orchestra) \u001b[0m │ \u001b[1;36m345\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m24\u001b[0m │ \u001b[32mClaudio Monteverdi\u001b[0m │ \u001b[1;36m66639\u001b[0m │ \u001b[1;36m1189062\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m1.110650\u001b[0m │\n", - "│ \u001b[1;36m3452\u001b[0m │ \u001b[32mSCRIABIN: Prelude in B Major, Op. 11, No. 11 \u001b[0m │ \u001b[1;36m318\u001b[0m │ \u001b[1;36m4\u001b[0m │ \u001b[1;36m24\u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[1;36m101293\u001b[0m │ \u001b[1;36m3819535\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m1.688217\u001b[0m │\n", - "│ \u001b[1;36m3448\u001b[0m │ \u001b[32mLamentations of Jeremiah, First Set \\ Incipit Lamentatio\u001b[0m │ \u001b[1;36m314\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m24\u001b[0m │ \u001b[32mThomas Tallis \u001b[0m │ \u001b[1;36m69194\u001b[0m │ \u001b[1;36m1208080\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m1.691350\u001b[0m │\n", - "│ \u001b[1;36m3492\u001b[0m │ \u001b[32mSing Joyfully \u001b[0m │ \u001b[1;36m314\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m24\u001b[0m │ \u001b[32mWilliam Byrd \u001b[0m │ \u001b[1;36m133768\u001b[0m │ \u001b[1;36m2256484\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m1.691350\u001b[0m │\n", - "└─────────┴──────────────────────────────────────────────────────────┴─────────┴─────────────┴─────────┴────────────────────┴──────────────┴─────────┴─────────────────┴─────────────────┘" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "track.group_by(\"AlbumId\").mutate(\n", " mean_mins_track=track.Milliseconds.mean() / 1e3 / 60\n", @@ -682,57 +483,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "39c80365", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
-       "┃ TrackId ┃ Name                                    ┃ AlbumId ┃ MediaTypeId ┃ GenreId ┃ Composer                                  ┃ Milliseconds ┃ Bytes    ┃ UnitPrice       ┃ mean_mins_track ┃\n",
-       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
-       "│ !int64  │ !string                                 │ int64   │ !int64      │ int64   │ string                                    │ !int64       │ int64    │ !decimal(10, 2) │ float64         │\n",
-       "├─────────┼─────────────────────────────────────────┼─────────┼─────────────┼─────────┼───────────────────────────────────────────┼──────────────┼──────────┼─────────────────┼─────────────────┤\n",
-       "│       1 │ For Those About To Rock (We Salute You) │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       343719 │ 11170334 │            0.99 │        4.000692 │\n",
-       "│       6 │ Put The Finger On You                   │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       205662 │  6713451 │            0.99 │        4.000692 │\n",
-       "│       7 │ Let's Get It Up                         │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       233926 │  7636561 │            0.99 │        4.000692 │\n",
-       "│       8 │ Inject The Venom                        │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       210834 │  6852860 │            0.99 │        4.000692 │\n",
-       "│       9 │ Snowballed                              │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       203102 │  6599424 │            0.99 │        4.000692 │\n",
-       "│      10 │ Evil Walks                              │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       263497 │  8611245 │            0.99 │        4.000692 │\n",
-       "│      11 │ C.O.D.                                  │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       199836 │  6566314 │            0.99 │        4.000692 │\n",
-       "│      12 │ Breaking The Rules                      │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       263288 │  8596840 │            0.99 │        4.000692 │\n",
-       "│      13 │ Night Of The Long Knives                │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       205688 │  6706347 │            0.99 │        4.000692 │\n",
-       "│      14 │ Spellbound                              │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson │       270863 │  8817038 │            0.99 │        4.000692 │\n",
-       "│       … │ …                                       │       … │           … │       … │ …                                         │            … │        … │               … │               … │\n",
-       "└─────────┴─────────────────────────────────────────┴─────────┴─────────────┴─────────┴───────────────────────────────────────────┴──────────────┴──────────┴─────────────────┴─────────────────┘\n",
-       "
\n" - ], - "text/plain": [ - "┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n", - "┃\u001b[1m \u001b[0m\u001b[1mTrackId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mName\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mAlbumId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMediaTypeId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mGenreId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mComposer\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMilliseconds\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mBytes\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mUnitPrice\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mmean_mins_track\u001b[0m\u001b[1m \u001b[0m┃\n", - "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n", - "│ \u001b[2m!int64\u001b[0m │ \u001b[2m!string\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2m!int64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2m!int64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2m!decimal(10, 2)\u001b[0m │ \u001b[2mfloat64\u001b[0m │\n", - "├─────────┼─────────────────────────────────────────┼─────────┼─────────────┼─────────┼───────────────────────────────────────────┼──────────────┼──────────┼─────────────────┼─────────────────┤\n", - "│ \u001b[1;36m1\u001b[0m │ \u001b[32mFor Those About To Rock (We Salute You)\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m343719\u001b[0m │ \u001b[1;36m11170334\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", - "│ \u001b[1;36m6\u001b[0m │ \u001b[32mPut The Finger On You \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m205662\u001b[0m │ \u001b[1;36m6713451\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", - "│ \u001b[1;36m7\u001b[0m │ \u001b[32mLet's Get It Up \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m233926\u001b[0m │ \u001b[1;36m7636561\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", - "│ \u001b[1;36m8\u001b[0m │ \u001b[32mInject The Venom \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m210834\u001b[0m │ \u001b[1;36m6852860\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", - "│ \u001b[1;36m9\u001b[0m │ \u001b[32mSnowballed \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m203102\u001b[0m │ \u001b[1;36m6599424\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", - "│ \u001b[1;36m10\u001b[0m │ \u001b[32mEvil Walks \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m263497\u001b[0m │ \u001b[1;36m8611245\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", - "│ \u001b[1;36m11\u001b[0m │ \u001b[32mC.O.D. \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m199836\u001b[0m │ \u001b[1;36m6566314\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", - "│ \u001b[1;36m12\u001b[0m │ \u001b[32mBreaking The Rules \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m263288\u001b[0m │ \u001b[1;36m8596840\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", - "│ \u001b[1;36m13\u001b[0m │ \u001b[32mNight Of The Long Knives \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m205688\u001b[0m │ \u001b[1;36m6706347\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", - "│ \u001b[1;36m14\u001b[0m │ \u001b[32mSpellbound \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson\u001b[0m │ \u001b[1;36m270863\u001b[0m │ \u001b[1;36m8817038\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[1;36m4.000692\u001b[0m │\n", - "│ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │\n", - "└─────────┴─────────────────────────────────────────┴─────────┴─────────────┴─────────┴───────────────────────────────────────────┴──────────────┴──────────┴─────────────────┴─────────────────┘" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "track.group_by(\"AlbumId\").mutate(mean_mins_track=track.Milliseconds.mean() / 1e3 / 60)" ] @@ -747,57 +501,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "02e6602a", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓\n",
-       "┃ TrackId ┃ Name                                    ┃ AlbumId ┃ MediaTypeId ┃ GenreId ┃ Composer                                                               ┃ Milliseconds ┃ Bytes    ┃ UnitPrice      ┃ Name_right ┃\n",
-       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩\n",
-       "│ int64   │ string                                  │ int64   │ int64       │ int64   │ string                                                                 │ int64        │ int64    │ decimal(10, 2) │ string     │\n",
-       "├─────────┼─────────────────────────────────────────┼─────────┼─────────────┼─────────┼────────────────────────────────────────────────────────────────────────┼──────────────┼──────────┼────────────────┼────────────┤\n",
-       "│       1 │ For Those About To Rock (We Salute You) │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson                              │       343719 │ 11170334 │           0.99 │ Rock       │\n",
-       "│       2 │ Balls to the Wall                       │       2 │           2 │       1 │ NULL                                                                   │       342562 │  5510424 │           0.99 │ Rock       │\n",
-       "│       3 │ Fast As a Shark                         │       3 │           2 │       1 │ F. Baltes, S. Kaufman, U. Dirkscneider & W. Hoffman                    │       230619 │  3990994 │           0.99 │ Rock       │\n",
-       "│       4 │ Restless and Wild                       │       3 │           2 │       1 │ F. Baltes, R.A. Smith-Diesel, S. Kaufman, U. Dirkscneider & W. Hoffman │       252051 │  4331779 │           0.99 │ Rock       │\n",
-       "│       5 │ Princess of the Dawn                    │       3 │           2 │       1 │ Deaffy & R.A. Smith-Diesel                                             │       375418 │  6290521 │           0.99 │ Rock       │\n",
-       "│       6 │ Put The Finger On You                   │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson                              │       205662 │  6713451 │           0.99 │ Rock       │\n",
-       "│       7 │ Let's Get It Up                         │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson                              │       233926 │  7636561 │           0.99 │ Rock       │\n",
-       "│       8 │ Inject The Venom                        │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson                              │       210834 │  6852860 │           0.99 │ Rock       │\n",
-       "│       9 │ Snowballed                              │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson                              │       203102 │  6599424 │           0.99 │ Rock       │\n",
-       "│      10 │ Evil Walks                              │       1 │           1 │       1 │ Angus Young, Malcolm Young, Brian Johnson                              │       263497 │  8611245 │           0.99 │ Rock       │\n",
-       "│       … │ …                                       │       … │           … │       … │ …                                                                      │            … │        … │              … │ …          │\n",
-       "└─────────┴─────────────────────────────────────────┴─────────┴─────────────┴─────────┴────────────────────────────────────────────────────────────────────────┴──────────────┴──────────┴────────────────┴────────────┘\n",
-       "
\n" - ], - "text/plain": [ - "┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓\n", - "┃\u001b[1m \u001b[0m\u001b[1mTrackId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mName\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mAlbumId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMediaTypeId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mGenreId\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mComposer\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mMilliseconds\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mBytes\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mUnitPrice\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mName_right\u001b[0m\u001b[1m \u001b[0m┃\n", - "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩\n", - "│ \u001b[2mint64\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mstring\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mint64\u001b[0m │ \u001b[2mdecimal(10, 2)\u001b[0m │ \u001b[2mstring\u001b[0m │\n", - "├─────────┼─────────────────────────────────────────┼─────────┼─────────────┼─────────┼────────────────────────────────────────────────────────────────────────┼──────────────┼──────────┼────────────────┼────────────┤\n", - "│ \u001b[1;36m1\u001b[0m │ \u001b[32mFor Those About To Rock (We Salute You)\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson \u001b[0m │ \u001b[1;36m343719\u001b[0m │ \u001b[1;36m11170334\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", - "│ \u001b[1;36m2\u001b[0m │ \u001b[32mBalls to the Wall \u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[2mNULL\u001b[0m │ \u001b[1;36m342562\u001b[0m │ \u001b[1;36m5510424\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", - "│ \u001b[1;36m3\u001b[0m │ \u001b[32mFast As a Shark \u001b[0m │ \u001b[1;36m3\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mF. Baltes, S. Kaufman, U. Dirkscneider & W. Hoffman \u001b[0m │ \u001b[1;36m230619\u001b[0m │ \u001b[1;36m3990994\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", - "│ \u001b[1;36m4\u001b[0m │ \u001b[32mRestless and Wild \u001b[0m │ \u001b[1;36m3\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mF. Baltes, R.A. Smith-Diesel, S. Kaufman, U. Dirkscneider & W. Hoffman\u001b[0m │ \u001b[1;36m252051\u001b[0m │ \u001b[1;36m4331779\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", - "│ \u001b[1;36m5\u001b[0m │ \u001b[32mPrincess of the Dawn \u001b[0m │ \u001b[1;36m3\u001b[0m │ \u001b[1;36m2\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mDeaffy & R.A. Smith-Diesel \u001b[0m │ \u001b[1;36m375418\u001b[0m │ \u001b[1;36m6290521\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", - "│ \u001b[1;36m6\u001b[0m │ \u001b[32mPut The Finger On You \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson \u001b[0m │ \u001b[1;36m205662\u001b[0m │ \u001b[1;36m6713451\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", - "│ \u001b[1;36m7\u001b[0m │ \u001b[32mLet's Get It Up \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson \u001b[0m │ \u001b[1;36m233926\u001b[0m │ \u001b[1;36m7636561\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", - "│ \u001b[1;36m8\u001b[0m │ \u001b[32mInject The Venom \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson \u001b[0m │ \u001b[1;36m210834\u001b[0m │ \u001b[1;36m6852860\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", - "│ \u001b[1;36m9\u001b[0m │ \u001b[32mSnowballed \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson \u001b[0m │ \u001b[1;36m203102\u001b[0m │ \u001b[1;36m6599424\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", - "│ \u001b[1;36m10\u001b[0m │ \u001b[32mEvil Walks \u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[1;36m1\u001b[0m │ \u001b[32mAngus Young, Malcolm Young, Brian Johnson \u001b[0m │ \u001b[1;36m263497\u001b[0m │ \u001b[1;36m8611245\u001b[0m │ \u001b[1;36m0.99\u001b[0m │ \u001b[32mRock \u001b[0m │\n", - "│ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │ \u001b[2m…\u001b[0m │\n", - "└─────────┴─────────────────────────────────────────┴─────────┴─────────────┴─────────┴────────────────────────────────────────────────────────────────────────┴──────────────┴──────────┴────────────────┴────────────┘" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "genre = connection.table(\"genre\")\n", "genre_and_track = track.inner_join(\n", @@ -850,7 +557,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "325c8789", "metadata": { "tags": [ @@ -870,7 +577,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "9117af26", "metadata": {}, "outputs": [], @@ -897,7 +604,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "e8277743", "metadata": {}, "outputs": [], @@ -918,7 +625,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "2097c995", "metadata": {}, "outputs": [], @@ -945,24 +652,10 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "14c14fc8", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[(1, 'Deadpond', 'Dive Wilson', None),\n", - " (2, 'Spider-Boy', 'Pedro Parqueador', None),\n", - " (3, 'Rusty-Man', 'Tommy Sharp', 48),\n", - " (4, 'Ms Amazing', 'Barjabeen Bhabra', 17)]" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "con = sqlite3.connect(Path(\"data/hero.db\"))\n", "\n", @@ -981,21 +674,10 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "afa69365", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "name='Deadpond' secret_name='Dive Wilson' id=1 age=None\n", - "name='Spider-Boy' secret_name='Pedro Parqueador' id=2 age=None\n", - "name='Rusty-Man' secret_name='Tommy Sharp' id=3 age=48\n", - "name='Ms Amazing' secret_name='Barjabeen Bhabra' id=4 age=17\n" - ] - } - ], + "outputs": [], "source": [ "from sqlmodel import select\n", "\n", @@ -1016,21 +698,10 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "8fae7081", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "name='Deadpond' secret_name='Dive Wilson' id=1 age=None\n", - "name='Spider-Boy' secret_name='Pedro Parqueador' id=2 age=None\n", - "name='Rusty-Man' secret_name='Tommy Sharp' id=3 age=48\n", - "name='Ms Amazing' secret_name='Barjabeen Bhabra' id=4 age=17\n" - ] - } - ], + "outputs": [], "source": [ "with Session(engine) as session:\n", " statement = select(Hero)\n", @@ -1050,19 +721,10 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "b2dbde03", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "name='Ms Amazing' secret_name='Barjabeen Bhabra' id=4 age=17\n", - "name='Rusty-Man' secret_name='Tommy Sharp' id=3 age=48\n" - ] - } - ], + "outputs": [], "source": [ "with Session(engine) as session:\n", " statement = select(Hero).where(Hero.age < 100).limit(2).order_by(Hero.age)\n", @@ -1082,7 +744,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "349e5d3d", "metadata": { "tags": [ diff --git a/dates-and-times.ipynb b/dates-and-times.ipynb index 1ef38ab..47b53ad 100644 --- a/dates-and-times.ipynb +++ b/dates-and-times.ipynb @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "51a55374", "metadata": { "tags": [ @@ -74,18 +74,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "84829a6b", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2026-04-28 13:36:00.720344\n" - ] - } - ], + "outputs": [], "source": [ "from datetime import datetime\n", "\n", @@ -129,18 +121,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "fc224a47", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2019-11-28 00:00:00\n" - ] - } - ], + "outputs": [], "source": [ "specific_datetime = datetime(2019, 11, 28)\n", "print(specific_datetime)" @@ -166,21 +150,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "4558d476", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "datetime.datetime(2002, 2, 16, 0, 0)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "date_string = \"16 February in 2002\"\n", "datetime.strptime(date_string, \"%d %B in %Y\")" @@ -196,21 +169,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "fb90ac84", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "datetime.datetime(2002, 2, 16, 0, 0)" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "date_string = \"16 Feb in 2002\"\n", "datetime.strptime(date_string, \"%d %b in %Y\")" @@ -226,19 +188,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "7a568f5b", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2002-02-03 00:00:00\n", - "2002-02-03 00:00:00\n" - ] - } - ], + "outputs": [], "source": [ "from dateutil.parser import parse\n", "\n", @@ -258,21 +211,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "d6b5e3c3", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Tuesday, 04, 2026'" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "now.strftime(\"%A, %m, %Y\")" ] @@ -331,21 +273,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "39ba17e6", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "now > specific_datetime" ] @@ -360,18 +291,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "fda57a44", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2309 days, 13:36:00.720344\n" - ] - } - ], + "outputs": [], "source": [ "time_diff = now - datetime(year=2020, month=1, day=1)\n", "print(time_diff)" @@ -387,21 +310,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "e1e25736", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "datetime.timedelta" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "type(time_diff)" ] @@ -449,21 +361,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "ed526fbc", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array('2020-01-01', dtype='datetime64[D]')" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import numpy as np\n", "\n", @@ -481,29 +382,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "537895c5", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array(['2020-01-01', '2020-01-02', '2020-01-03', '2020-01-04',\n", - " '2020-01-05', '2020-01-06', '2020-01-07', '2020-01-08',\n", - " '2020-01-09', '2020-01-10', '2020-01-11', '2020-01-12',\n", - " '2020-01-13', '2020-01-14', '2020-01-15', '2020-01-16',\n", - " '2020-01-17', '2020-01-18', '2020-01-19', '2020-01-20',\n", - " '2020-01-21', '2020-01-22', '2020-01-23', '2020-01-24',\n", - " '2020-01-25', '2020-01-26', '2020-01-27', '2020-01-28',\n", - " '2020-01-29', '2020-01-30', '2020-01-31', '2020-02-01'],\n", - " dtype='datetime64[D]')" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "date + range(32)" ] @@ -520,21 +402,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "cd7a15e3", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "np.datetime64('2020-01-01T09:00')" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "np.datetime64(\"2020-01-01 09:00\")" ] @@ -565,21 +436,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "48f0e9c6", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Timestamp('2020-02-16 00:00:00')" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import pandas as pd\n", "\n", @@ -609,24 +469,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "462b26da", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "DatetimeIndex(['2020-02-16', '2020-02-17', '2020-02-18', '2020-02-19',\n", - " '2020-02-20', '2020-02-21', '2020-02-22', '2020-02-23',\n", - " '2020-02-24', '2020-02-25', '2020-02-26', '2020-02-27'],\n", - " dtype='datetime64[ns]', freq=None)" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "date + pd.to_timedelta(np.arange(12), \"D\")" ] @@ -643,23 +489,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "10e71325", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "DatetimeIndex(['2018-01-01', '2018-01-02', '2018-01-03', '2018-01-04',\n", - " '2018-01-05', '2018-01-06', '2018-01-07', '2018-01-08'],\n", - " dtype='datetime64[ns]', freq='D')" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pd.date_range(start=\"2018/1/1\", end=\"2018/1/8\")" ] @@ -674,23 +507,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "291ace2c", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "DatetimeIndex(['2018-01-01 00:00:00', '2018-01-01 01:00:00',\n", - " '2018-01-01 02:00:00'],\n", - " dtype='datetime64[ns]', freq='h')" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pd.date_range(\"2018-01-01\", periods=3, freq=\"h\")" ] @@ -705,23 +525,10 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "6703682c", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "DatetimeIndex(['2017-12-31 16:00:00-08:00', '2017-12-31 17:00:00-08:00',\n", - " '2017-12-31 18:00:00-08:00'],\n", - " dtype='datetime64[ns, US/Pacific]', freq='h')" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dti = pd.date_range(\"2018-01-01\", periods=3, freq=\"h\").tz_localize(\"UTC\")\n", "dti.tz_convert(\"US/Pacific\")" @@ -737,79 +544,10 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "dd00df7f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
dateVacancies (ICT), thousands
02001 MAY568
12001 JUN563
22001 JUL554
32001 AUG554
42001 SEP536
\n", - "
" - ], - "text/plain": [ - " date Vacancies (ICT), thousands\n", - "0 2001 MAY 568\n", - "1 2001 JUN 563\n", - "2 2001 JUL 554\n", - "3 2001 AUG 554\n", - "4 2001 SEP 536" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import requests\n", "\n", @@ -834,26 +572,10 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "247d9725", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "RangeIndex: 281 entries, 0 to 280\n", - "Data columns (total 2 columns):\n", - " # Column Non-Null Count Dtype \n", - "--- ------ -------------- ----- \n", - " 0 date 281 non-null object\n", - " 1 Vacancies (ICT), thousands 281 non-null int64 \n", - "dtypes: int64(1), object(1)\n", - "memory usage: 4.5+ KB\n" - ] - } - ], + "outputs": [], "source": [ "df.info()" ] @@ -868,34 +590,10 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "b90f8038", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60965/3535541307.py:1: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.\n", - " df[\"date\"] = pd.to_datetime(df[\"date\"])\n" - ] - }, - { - "data": { - "text/plain": [ - "0 2001-05-01\n", - "1 2001-06-01\n", - "2 2001-07-01\n", - "3 2001-08-01\n", - "4 2001-09-01\n", - "Name: date, dtype: datetime64[ns]" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[\"date\"] = pd.to_datetime(df[\"date\"])\n", "df[\"date\"].head()" @@ -913,23 +611,10 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "05d056ae", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 1, '19, 22\n", - "1 1, '19, 23\n", - "Name: date, dtype: object" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "small_df = pd.DataFrame({\"date\": [\"1, '19, 22\", \"1, '19, 23\"], \"values\": [\"1\", \"2\"]})\n", "small_df[\"date\"]" @@ -945,23 +630,10 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "514c9052", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 2019-01-22\n", - "1 2019-01-23\n", - "Name: date, dtype: datetime64[ns]" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pd.to_datetime(small_df[\"date\"], format=\"%m, '%y, %d\")" ] @@ -978,79 +650,10 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "ac3addbc", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
dateVacancies (ICT), thousands
02001-05-31568
12001-06-30563
22001-07-31554
32001-08-31554
42001-09-30536
\n", - "
" - ], - "text/plain": [ - " date Vacancies (ICT), thousands\n", - "0 2001-05-31 568\n", - "1 2001-06-30 563\n", - "2 2001-07-31 554\n", - "3 2001-08-31 554\n", - "4 2001-09-30 536" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[\"date\"] = df[\"date\"] + pd.offsets.MonthEnd()\n", "df.head()" @@ -1076,38 +679,10 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "a6c3d2d9", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using `dt.day_name()`\n", - "0 Thursday\n", - "1 Saturday\n", - "2 Tuesday\n", - "3 Friday\n", - "4 Sunday\n", - "Name: date, dtype: object\n", - "Using `dt.isocalendar()`\n", - " year week day\n", - "0 2001 22 4\n", - "1 2001 26 6\n", - "2 2001 31 2\n", - "3 2001 35 5\n", - "4 2001 39 7\n", - "Using `dt.month`\n", - "0 5\n", - "1 6\n", - "2 7\n", - "3 8\n", - "4 9\n", - "Name: date, dtype: int32\n" - ] - } - ], + "outputs": [], "source": [ "print(\"Using `dt.day_name()`\")\n", "print(df[\"date\"].dt.day_name().head())\n", @@ -1129,78 +704,10 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "id": "e0a4f68d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Vacancies (ICT), thousands
date
2001-05-31568
2001-06-30563
2001-07-31554
2001-08-31554
2001-09-30536
\n", - "
" - ], - "text/plain": [ - " Vacancies (ICT), thousands\n", - "date \n", - "2001-05-31 568\n", - "2001-06-30 563\n", - "2001-07-31 554\n", - "2001-08-31 554\n", - "2001-09-30 536" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = df.set_index(\"date\")\n", "df.head()" @@ -1216,23 +723,10 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "acf1ae60", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "DatetimeIndex(['2001-05-31', '2001-06-30', '2001-07-31', '2001-08-31',\n", - " '2001-09-30'],\n", - " dtype='datetime64[ns]', name='date', freq=None)" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.index[:5]" ] @@ -1247,31 +741,10 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "9146c99d", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60965/2067773505.py:1: FutureWarning: 'M' is deprecated and will be removed in a future version, please use 'ME' instead.\n", - " df = df.asfreq(\"M\")\n" - ] - }, - { - "data": { - "text/plain": [ - "DatetimeIndex(['2001-05-31', '2001-06-30', '2001-07-31', '2001-08-31',\n", - " '2001-09-30'],\n", - " dtype='datetime64[ns]', name='date', freq='ME')" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = df.asfreq(\"M\")\n", "df.index[:5]" @@ -1321,1352 +794,10 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "b4c5f841", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:36:02.756256\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "df.plot();" ] @@ -2687,181 +818,10 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "id": "e56ba5c4", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60965/311401334.py:1: FutureWarning: 'A' is deprecated and will be removed in a future version, please use 'YE' instead.\n", - " df.resample(\"A\").mean()\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Vacancies (ICT), thousands
date
2001-12-31540.625000
2002-12-31517.500000
2003-12-31504.166667
2004-12-31551.916667
2005-12-31544.666667
2006-12-31529.500000
2007-12-31576.333333
2008-12-31544.583333
2009-12-31402.750000
2010-12-31424.166667
2011-12-31413.250000
2012-12-31423.916667
2013-12-31480.250000
2014-12-31592.416667
2015-12-31655.166667
2016-12-31671.250000
2017-12-31704.750000
2018-12-31742.666667
2019-12-31734.166667
2020-12-31487.500000
2021-12-31843.416667
2022-12-311092.083333
2023-12-31894.500000
2024-12-31767.888889
\n", - "
" - ], - "text/plain": [ - " Vacancies (ICT), thousands\n", - "date \n", - "2001-12-31 540.625000\n", - "2002-12-31 517.500000\n", - "2003-12-31 504.166667\n", - "2004-12-31 551.916667\n", - "2005-12-31 544.666667\n", - "2006-12-31 529.500000\n", - "2007-12-31 576.333333\n", - "2008-12-31 544.583333\n", - "2009-12-31 402.750000\n", - "2010-12-31 424.166667\n", - "2011-12-31 413.250000\n", - "2012-12-31 423.916667\n", - "2013-12-31 480.250000\n", - "2014-12-31 592.416667\n", - "2015-12-31 655.166667\n", - "2016-12-31 671.250000\n", - "2017-12-31 704.750000\n", - "2018-12-31 742.666667\n", - "2019-12-31 734.166667\n", - "2020-12-31 487.500000\n", - "2021-12-31 843.416667\n", - "2022-12-31 1092.083333\n", - "2023-12-31 894.500000\n", - "2024-12-31 767.888889" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.resample(\"A\").mean()" ] @@ -2876,94 +836,10 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "id": "fbbbcdff", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Vacancies (ICT), thousands
meanstd
date
2001-12-31540.62500022.398581
2006-12-31529.55000020.434621
2011-12-31472.21666777.919796
2016-12-31564.60000099.829210
2021-12-31702.500000164.019480
\n", - "
" - ], - "text/plain": [ - " Vacancies (ICT), thousands \n", - " mean std\n", - "date \n", - "2001-12-31 540.625000 22.398581\n", - "2006-12-31 529.550000 20.434621\n", - "2011-12-31 472.216667 77.919796\n", - "2016-12-31 564.600000 99.829210\n", - "2021-12-31 702.500000 164.019480" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.resample(\"5YE\").agg([\"mean\", \"std\"]).head()" ] @@ -2978,111 +854,10 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "id": "9a48a45f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Vacancies (ICT), thousands
date
2001-05-31568.0
2001-06-01NaN
2001-06-02NaN
2001-06-03NaN
2001-06-04NaN
......
2024-09-26NaN
2024-09-27NaN
2024-09-28NaN
2024-09-29NaN
2024-09-30727.0
\n", - "

8524 rows × 1 columns

\n", - "
" - ], - "text/plain": [ - " Vacancies (ICT), thousands\n", - "date \n", - "2001-05-31 568.0\n", - "2001-06-01 NaN\n", - "2001-06-02 NaN\n", - "2001-06-03 NaN\n", - "2001-06-04 NaN\n", - "... ...\n", - "2024-09-26 NaN\n", - "2024-09-27 NaN\n", - "2024-09-28 NaN\n", - "2024-09-29 NaN\n", - "2024-09-30 727.0\n", - "\n", - "[8524 rows x 1 columns]" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.resample(\"D\").asfreq()" ] @@ -3097,83 +872,10 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "id": "d3ac1789", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Vacancies (ICT), thousands
date
2001-05-31568.000000
2001-06-01567.833333
2001-06-02567.666667
2001-06-03567.500000
2001-06-04NaN
2001-06-05NaN
\n", - "
" - ], - "text/plain": [ - " Vacancies (ICT), thousands\n", - "date \n", - "2001-05-31 568.000000\n", - "2001-06-01 567.833333\n", - "2001-06-02 567.666667\n", - "2001-06-03 567.500000\n", - "2001-06-04 NaN\n", - "2001-06-05 NaN" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.resample(\"D\").interpolate(method=\"linear\", limit_direction=\"forward\", limit=3)[:6]" ] @@ -3188,26 +890,10 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "id": "51647c56", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[*********************100%***********************] 1 of 1 completed" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], + "outputs": [], "source": [ "# Get stock market data\n", "import yfinance as yf\n", @@ -3219,1640 +905,20 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "id": "ab1efae3", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
'color'
'#bc80bd'
'#fb8072'
'#b3de69'
'#fdb462'
'#fccde5'
'#8dd3c7'
'#ffed6f'
'#bebada'
'#80b1d3'
'#ccebc5'
'#d9d9d9'
" - ], - "text/plain": [ - "cycler('color', ['#bc80bd', '#fb8072', '#b3de69', '#fdb462', '#fccde5', '#8dd3c7', '#ffed6f', '#bebada', '#80b1d3', '#ccebc5', '#d9d9d9'])" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "plt.rcParams[\"axes.prop_cycle\"]" ] }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "id": "fa0c9973", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:36:08.720742\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "from itertools import cycle\n", "\n", @@ -4891,111 +957,10 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "id": "1ddc4fb2", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Vacancies (ICT), thousands
date
2001-05-31NaN
2001-06-30565.5
2001-07-31558.5
2001-08-31554.0
2001-09-30545.0
......
2024-05-31776.5
2024-06-30760.0
2024-07-31748.0
2024-08-31737.0
2024-09-30729.5
\n", - "

281 rows × 1 columns

\n", - "
" - ], - "text/plain": [ - " Vacancies (ICT), thousands\n", - "date \n", - "2001-05-31 NaN\n", - "2001-06-30 565.5\n", - "2001-07-31 558.5\n", - "2001-08-31 554.0\n", - "2001-09-30 545.0\n", - "... ...\n", - "2024-05-31 776.5\n", - "2024-06-30 760.0\n", - "2024-07-31 748.0\n", - "2024-08-31 737.0\n", - "2024-09-30 729.5\n", - "\n", - "[281 rows x 1 columns]" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.rolling(2).mean()" ] @@ -5018,111 +983,10 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "id": "0ea9c8ce", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Vacancies (ICT), thousands
date
2001-05-31568.000000
2001-06-30565.222222
2001-07-31560.622951
2001-08-31558.379404
2001-09-30551.722037
......
2024-05-31813.183347
2024-06-30801.346677
2024-07-31789.477342
2024-08-31777.981873
2024-09-30767.785499
\n", - "

281 rows × 1 columns

\n", - "
" - ], - "text/plain": [ - " Vacancies (ICT), thousands\n", - "date \n", - "2001-05-31 568.000000\n", - "2001-06-30 565.222222\n", - "2001-07-31 560.622951\n", - "2001-08-31 558.379404\n", - "2001-09-30 551.722037\n", - "... ...\n", - "2024-05-31 813.183347\n", - "2024-06-30 801.346677\n", - "2024-07-31 789.477342\n", - "2024-08-31 777.981873\n", - "2024-09-30 767.785499\n", - "\n", - "[281 rows x 1 columns]" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.ewm(alpha=0.2).mean()" ] @@ -5137,3078 +1001,10 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "id": "0af7b5e6", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:36:08.899458\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "fig, ax = plt.subplots()\n", "roll_num = 28\n", @@ -8235,2366 +1031,10 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "id": "134199ae", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:36:08.991604\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "roll = xf[\"Close\"].rolling(50, center=True)\n", "\n", @@ -10617,92 +1057,10 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "id": "3078fbb4", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Vacancies (ICT), thousandslead (12 months)lag (3 months)
date
2001-05-31568518.0NaN
2001-06-30563514.0NaN
2001-07-31554517.0NaN
2001-08-31554517.0568.0
2001-09-30536519.0563.0
\n", - "
" - ], - "text/plain": [ - " Vacancies (ICT), thousands lead (12 months) lag (3 months)\n", - "date \n", - "2001-05-31 568 518.0 NaN\n", - "2001-06-30 563 514.0 NaN\n", - "2001-07-31 554 517.0 NaN\n", - "2001-08-31 554 517.0 568.0\n", - "2001-09-30 536 519.0 563.0" - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "lead = 12\n", "lag = 3\n", @@ -10714,1831 +1072,10 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "id": "18b9afb3", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:36:09.071401\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "df.iloc[100:300, :].plot();" ] diff --git a/exploratory-data-analysis.ipynb b/exploratory-data-analysis.ipynb index b08137e..071a96b 100644 --- a/exploratory-data-analysis.ipynb +++ b/exploratory-data-analysis.ipynb @@ -28,51 +28,10 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "a3377aa6", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "import pandas as pd\n", "from lets_plot import *\n", @@ -85,7 +44,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "51a55374", "metadata": { "tags": [ @@ -143,127 +102,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "069caa7c", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
caratcutcolorclaritydepthtablepricexyz
00.23IdealESI261.555.03263.953.982.43
10.21PremiumESI159.861.03263.893.842.31
20.23GoodEVS156.965.03274.054.072.31
30.29PremiumIVS262.458.03344.204.232.63
40.31GoodJSI263.358.03354.344.352.75
\n", - "
" - ], - "text/plain": [ - " carat cut color clarity depth table price x y z\n", - "0 0.23 Ideal E SI2 61.5 55.0 326 3.95 3.98 2.43\n", - "1 0.21 Premium E SI1 59.8 61.0 326 3.89 3.84 2.31\n", - "2 0.23 Good E VS1 56.9 65.0 327 4.05 4.07 2.31\n", - "3 0.29 Premium I VS2 62.4 58.0 334 4.20 4.23 2.63\n", - "4 0.31 Good J SI2 63.3 58.0 335 4.34 4.35 2.75" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "diamonds = pd.read_csv(\n", " \"https://github.com/mwaskom/seaborn-data/raw/master/diamonds.csv\"\n", @@ -289,135 +131,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "97900f58", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(diamonds, aes(x=\"carat\")) + geom_histogram(binwidth=0.5))" ] @@ -455,135 +172,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "20d75550", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "smaller_diamonds = diamonds.query(\"carat < 3\").copy()\n", "\n", @@ -633,135 +225,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "d9d7e995", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(diamonds, aes(x=\"y\")) + geom_histogram(binwidth=0.5))" ] @@ -777,140 +244,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "ea8f8bf3", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(diamonds, aes(x=\"y\"))\n", @@ -932,123 +269,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "e81ffb55", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
xyzprice
119630.000.00.005139
159510.000.00.006381
240678.0958.98.0612210
245200.000.00.0012800
262430.000.00.0015686
274290.000.00.0018034
491895.1531.85.122075
495560.000.00.002130
495570.000.00.002130
\n", - "
" - ], - "text/plain": [ - " x y z price\n", - "11963 0.00 0.0 0.00 5139\n", - "15951 0.00 0.0 0.00 6381\n", - "24067 8.09 58.9 8.06 12210\n", - "24520 0.00 0.0 0.00 12800\n", - "26243 0.00 0.0 0.00 15686\n", - "27429 0.00 0.0 0.00 18034\n", - "49189 5.15 31.8 5.12 2075\n", - "49556 0.00 0.0 0.00 2130\n", - "49557 0.00 0.0 0.00 2130" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "unusual = diamonds.query(\"y < 3 or y > 20\").loc[:, [\"x\", \"y\", \"z\", \"price\"]]\n", "unusual" @@ -1120,7 +344,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "ecf345a7", "metadata": {}, "outputs": [], @@ -1140,135 +364,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "15a43255", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(diamonds2, aes(x=\"x\", y=\"y\")) + geom_point())" ] @@ -1286,195 +385,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "0a4ea922", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
0201311517.05152.0830.081911.0UA1545N14228EWRIAH227.014005152013-01-01T10:00:00Z
1201311533.05294.0850.083020.0UA1714N24211LGAIAH227.014165292013-01-01T10:00:00Z
2201311542.05402.0923.085033.0AA1141N619AAJFKMIA160.010895402013-01-01T10:00:00Z
3201311544.0545-1.01004.01022-18.0B6725N804JBJFKBQN183.015765452013-01-01T10:00:00Z
4201311554.0600-6.0812.0837-25.0DL461N668DNLGAATL116.0762602013-01-01T11:00:00Z
\n", - "
" - ], - "text/plain": [ - " year month day dep_time sched_dep_time dep_delay arr_time \\\n", - "0 2013 1 1 517.0 515 2.0 830.0 \n", - "1 2013 1 1 533.0 529 4.0 850.0 \n", - "2 2013 1 1 542.0 540 2.0 923.0 \n", - "3 2013 1 1 544.0 545 -1.0 1004.0 \n", - "4 2013 1 1 554.0 600 -6.0 812.0 \n", - "\n", - " sched_arr_time arr_delay carrier flight tailnum origin dest air_time \\\n", - "0 819 11.0 UA 1545 N14228 EWR IAH 227.0 \n", - "1 830 20.0 UA 1714 N24211 LGA IAH 227.0 \n", - "2 850 33.0 AA 1141 N619AA JFK MIA 160.0 \n", - "3 1022 -18.0 B6 725 N804JB JFK BQN 183.0 \n", - "4 837 -25.0 DL 461 N668DN LGA ATL 116.0 \n", - "\n", - " distance hour minute time_hour \n", - "0 1400 5 15 2013-01-01T10:00:00Z \n", - "1 1416 5 29 2013-01-01T10:00:00Z \n", - "2 1089 5 40 2013-01-01T10:00:00Z \n", - "3 1576 5 45 2013-01-01T10:00:00Z \n", - "4 762 6 0 2013-01-01T11:00:00Z " - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "url = \"https://raw.githubusercontent.com/byuidatascience/data4python4ds/master/data-raw/flights/flights.csv\"\n", "flights = pd.read_csv(url)\n", @@ -1483,164 +397,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "6849f4d9", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights2 = flights.assign(\n", " cancelled=lambda x: pd.isna(x[\"dep_time\"]),\n", @@ -1692,138 +452,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "e1719d8f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(diamonds, aes(x=\"price\"))\n", @@ -1844,140 +476,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "9388e24b", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(diamonds, aes(x=\"price\"))\n", @@ -1997,163 +499,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "a3f333a6", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(diamonds, aes(x=\"cut\", y=\"price\")) + geom_boxplot())" ] @@ -2172,150 +521,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "6949db81", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "mpg = pd.read_csv(\n", " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/mpg.csv\", index_col=0\n", @@ -2335,168 +544,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "a5b1ed09", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg) + geom_boxplot(aes(as_discrete(\"class\", order_by=\"..middle..\"), \"hwy\")))" ] @@ -2511,172 +562,10 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "920a4268", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg)\n", @@ -2719,7 +608,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "68d330d2", "metadata": {}, "outputs": [], @@ -2741,112 +630,10 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "e858cd22", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(ct_cut_color, aes(x=\"color\", y=\"cut\")) + geom_tile(aes(fill=\"value\")))" ] @@ -2880,135 +667,10 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "2afe2535", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(smaller_diamonds, aes(x=\"carat\", y=\"price\")) + geom_point())" ] @@ -3026,136 +688,10 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "b55707a9", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(smaller_diamonds, aes(x=\"carat\", y=\"price\")) + geom_point(alpha=1 / 20))" ] @@ -3184,152 +720,10 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "13079065", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
caratdepthtablepricexyz
count53940.00000053940.00000053940.00000053940.00000053940.00000053940.00000053940.000000
mean0.79794061.74940557.4571843932.7997225.7311575.7345263.538734
std0.4740111.4326212.2344913989.4397381.1217611.1421350.705699
min0.20000043.00000043.000000326.0000000.0000000.0000000.000000
25%0.40000061.00000056.000000950.0000004.7100004.7200002.910000
50%0.70000061.80000057.0000002401.0000005.7000005.7100003.530000
75%1.04000062.50000059.0000005324.2500006.5400006.5400004.040000
max5.01000079.00000095.00000018823.00000010.74000058.90000031.800000
\n", - "
" - ], - "text/plain": [ - " carat depth table price x \\\n", - "count 53940.000000 53940.000000 53940.000000 53940.000000 53940.000000 \n", - "mean 0.797940 61.749405 57.457184 3932.799722 5.731157 \n", - "std 0.474011 1.432621 2.234491 3989.439738 1.121761 \n", - "min 0.200000 43.000000 43.000000 326.000000 0.000000 \n", - "25% 0.400000 61.000000 56.000000 950.000000 4.710000 \n", - "50% 0.700000 61.800000 57.000000 2401.000000 5.700000 \n", - "75% 1.040000 62.500000 59.000000 5324.250000 6.540000 \n", - "max 5.010000 79.000000 95.000000 18823.000000 10.740000 \n", - "\n", - " y z \n", - "count 53940.000000 53940.000000 \n", - "mean 5.734526 3.538734 \n", - "std 1.142135 0.705699 \n", - "min 0.000000 0.000000 \n", - "25% 4.720000 2.910000 \n", - "50% 5.710000 3.530000 \n", - "75% 6.540000 4.040000 \n", - "max 58.900000 31.800000 " - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "diamonds.describe()" ] @@ -3344,142 +738,10 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "b4144440", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
caratdepthtablepricexyz
count53940.053940.053940.053940.053940.053940.053940.0
mean0.861.757.53932.85.75.73.5
std0.51.42.23989.41.11.10.7
min0.243.043.0326.00.00.00.0
25%0.461.056.0950.04.74.72.9
50%0.761.857.02401.05.75.73.5
75%1.062.559.05324.26.56.54.0
max5.079.095.018823.010.758.931.8
\n", - "
" - ], - "text/plain": [ - " carat depth table price x y z\n", - "count 53940.0 53940.0 53940.0 53940.0 53940.0 53940.0 53940.0\n", - "mean 0.8 61.7 57.5 3932.8 5.7 5.7 3.5\n", - "std 0.5 1.4 2.2 3989.4 1.1 1.1 0.7\n", - "min 0.2 43.0 43.0 326.0 0.0 0.0 0.0\n", - "25% 0.4 61.0 56.0 950.0 4.7 4.7 2.9\n", - "50% 0.7 61.8 57.0 2401.0 5.7 5.7 3.5\n", - "75% 1.0 62.5 59.0 5324.2 6.5 6.5 4.0\n", - "max 5.0 79.0 95.0 18823.0 10.7 58.9 31.8" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "sum_table = diamonds.describe().round(1)\n", "sum_table" @@ -3495,139 +757,10 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "cd2f8772", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
countmeanstdmin25%50%75%max
carat53940.00.80.50.20.40.71.05.0
depth53940.061.71.443.061.061.862.579.0
table53940.057.52.243.056.057.059.095.0
price53940.03932.83989.4326.0950.02401.05324.218823.0
x53940.05.71.10.04.75.76.510.7
y53940.05.71.10.04.75.76.558.9
z53940.03.50.70.02.93.54.031.8
\n", - "
" - ], - "text/plain": [ - " count mean std min 25% 50% 75% max\n", - "carat 53940.0 0.8 0.5 0.2 0.4 0.7 1.0 5.0\n", - "depth 53940.0 61.7 1.4 43.0 61.0 61.8 62.5 79.0\n", - "table 53940.0 57.5 2.2 43.0 56.0 57.0 59.0 95.0\n", - "price 53940.0 3932.8 3989.4 326.0 950.0 2401.0 5324.2 18823.0\n", - "x 53940.0 5.7 1.1 0.0 4.7 5.7 6.5 10.7\n", - "y 53940.0 5.7 1.1 0.0 4.7 5.7 6.5 58.9\n", - "z 53940.0 3.5 0.7 0.0 2.9 3.5 4.0 31.8" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "sum_table = sum_table.T\n", "sum_table" @@ -3647,110 +780,10 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "id": "5afcacbc", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_58745/1534868389.py:2: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n", - " diamonds.groupby([\"cut\", \"color\"])[\"price\"]\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Sale price (thousands)
colorDEFGHIJ
cut       
Fair4.293.683.834.245.144.694.98
Good3.413.423.504.124.285.084.57
Very Good3.473.213.783.874.545.265.10
Premium3.633.544.324.505.225.956.29
Ideal2.632.603.373.723.894.454.92
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " diamonds.groupby([\"cut\", \"color\"])[\"price\"]\n", @@ -3777,203 +810,10 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "21e65189", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
cutFairGoodVery GoodPremiumIdeal
color     
D163662151316032834
E224933240023373903
F312909216423313826
G314871229929244884
H303702182423603115
I175522120414282093
J119307678808896
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pd.crosstab(diamonds[\"color\"], diamonds[\"cut\"]).style.background_gradient(cmap=\"plasma\")" ] @@ -3996,223 +836,10 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "bb0162ba", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
cutFairGoodVery GoodPremiumIdeal
color     
D163662151316032834
E224933240023373903
F312909216423313826
G314871229929244884
H303702182423603115
I175522120414282093
J119307678808896
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " pd.crosstab(diamonds[\"color\"], diamonds[\"cut\"])\n", @@ -4231,106 +858,10 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "5d19072c", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
cutFairGoodVery GoodPremiumIdeal
color     
D163662151316032834
E224933240023373903
F312909216423313826
G314871229929244884
H303702182423603115
I175522120414282093
J119307678808896
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pd.crosstab(diamonds[\"color\"], diamonds[\"cut\"]).style.highlight_max().format(\"{:.0f}\")" ] @@ -4355,165 +886,10 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "id": "b479d5b1", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
pickupdropoffpassengersdistancefaretiptollstotalcolorpaymentpickup_zonedropoff_zonepickup_boroughdropoff_borough
02019-03-23 20:21:092019-03-23 20:27:2411.607.02.150.012.95yellowcredit cardLenox Hill WestUN/Turtle Bay SouthManhattanManhattan
12019-03-04 16:11:552019-03-04 16:19:0010.795.00.000.09.30yellowcashUpper West Side SouthUpper West Side SouthManhattanManhattan
22019-03-27 17:53:012019-03-27 18:00:2511.377.52.360.014.16yellowcredit cardAlphabet CityWest VillageManhattanManhattan
32019-03-10 01:23:592019-03-10 01:49:5117.7027.06.150.036.95yellowcredit cardHudson SqYorkville WestManhattanManhattan
42019-03-30 13:27:422019-03-30 13:37:1432.169.01.100.013.40yellowcredit cardMidtown EastYorkville WestManhattanManhattan
\n", - "
" - ], - "text/plain": [ - " pickup dropoff passengers distance fare tip \\\n", - "0 2019-03-23 20:21:09 2019-03-23 20:27:24 1 1.60 7.0 2.15 \n", - "1 2019-03-04 16:11:55 2019-03-04 16:19:00 1 0.79 5.0 0.00 \n", - "2 2019-03-27 17:53:01 2019-03-27 18:00:25 1 1.37 7.5 2.36 \n", - "3 2019-03-10 01:23:59 2019-03-10 01:49:51 1 7.70 27.0 6.15 \n", - "4 2019-03-30 13:27:42 2019-03-30 13:37:14 3 2.16 9.0 1.10 \n", - "\n", - " tolls total color payment pickup_zone \\\n", - "0 0.0 12.95 yellow credit card Lenox Hill West \n", - "1 0.0 9.30 yellow cash Upper West Side South \n", - "2 0.0 14.16 yellow credit card Alphabet City \n", - "3 0.0 36.95 yellow credit card Hudson Sq \n", - "4 0.0 13.40 yellow credit card Midtown East \n", - "\n", - " dropoff_zone pickup_borough dropoff_borough \n", - "0 UN/Turtle Bay South Manhattan Manhattan \n", - "1 Upper West Side South Manhattan Manhattan \n", - "2 West Village Manhattan Manhattan \n", - "3 Yorkville West Manhattan Manhattan \n", - "4 Yorkville West Manhattan Manhattan " - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "taxis = pd.read_csv(\"https://github.com/mwaskom/seaborn-data/raw/master/taxis.csv\")\n", "# turn the pickup time column into a datetime\n", @@ -4536,1289 +912,20 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "id": "ee971c9c", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "RangeIndex: 6433 entries, 0 to 6432\n", - "Data columns (total 14 columns):\n", - " # Column Non-Null Count Dtype \n", - "--- ------ -------------- ----- \n", - " 0 pickup 6433 non-null datetime64[ns]\n", - " 1 dropoff 6433 non-null datetime64[ns]\n", - " 2 passengers 6433 non-null int64 \n", - " 3 distance 6433 non-null float64 \n", - " 4 fare 6433 non-null float64 \n", - " 5 tip 6433 non-null float64 \n", - " 6 tolls 6433 non-null float64 \n", - " 7 total 6433 non-null float64 \n", - " 8 color 6433 non-null category \n", - " 9 payment 6389 non-null category \n", - " 10 pickup_zone 6407 non-null string \n", - " 11 dropoff_zone 6388 non-null string \n", - " 12 pickup_borough 6407 non-null category \n", - " 13 dropoff_borough 6388 non-null category \n", - "dtypes: category(4), datetime64[ns](2), float64(5), int64(1), string(2)\n", - "memory usage: 528.5 KB\n" - ] - } - ], + "outputs": [], "source": [ "taxis.info()" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "id": "2015b1dc", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:34:43.401430\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "(\n", " taxis.set_index(\"pickup\")\n", @@ -5842,1476 +949,10 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "id": "51e86185", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:34:43.501760\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "(\n", " taxis.set_index(\"pickup\")\n", @@ -7344,888 +985,10 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "id": "79ceca92", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:34:43.567782\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "taxis.value_counts(\"payment\").sort_index().plot.bar(title=\"Counts\", rot=0);" ] @@ -8240,1011 +1003,10 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "id": "5efc5817", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:34:43.648471\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "taxis[\"tip\"].plot.hist(bins=30, title=\"Tip\");" ] @@ -9259,2055 +1021,10 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "id": "0b735d15", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:34:43.718910\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "(taxis[[\"fare\", \"tolls\", \"tip\"]].plot.box());" ] @@ -11322,7210 +1039,10 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "id": "66adada2", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:34:43.811014\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "taxis.plot.scatter(x=\"fare\", y=\"tip\", alpha=0.7, ylim=(0, None));" ] @@ -18554,132 +1071,10 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "id": "32796b5f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
╭──────────────────────────────────────────────── skimpy summary ─────────────────────────────────────────────────╮\n",
-       "│          Data Summary                Data Types               Categories                                        │\n",
-       "│ ┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┓ ┏━━━━━━━━━━━━━┳━━━━━━━┓ ┏━━━━━━━━━━━━━━━━━━━━━━━┓                                │\n",
-       "│ ┃ Dataframe         ┃ Values ┃ ┃ Column Type ┃ Count ┃ ┃ Categorical Variables ┃                                │\n",
-       "│ ┡━━━━━━━━━━━━━━━━━━━╇━━━━━━━━┩ ┡━━━━━━━━━━━━━╇━━━━━━━┩ ┡━━━━━━━━━━━━━━━━━━━━━━━┩                                │\n",
-       "│ │ Number of rows    │ 6433   │ │ float64     │ 5     │ │ color                 │                                │\n",
-       "│ │ Number of columns │ 14     │ │ category    │ 4     │ │ payment               │                                │\n",
-       "│ └───────────────────┴────────┘ │ datetime64  │ 2     │ │ pickup_borough        │                                │\n",
-       "│                                │ string      │ 2     │ │ dropoff_borough       │                                │\n",
-       "│                                │ int64       │ 1     │ └───────────────────────┘                                │\n",
-       "│                                └─────────────┴───────┘                                                          │\n",
-       "│                                                     number                                                      │\n",
-       "│ ┏━━━━━━━━━━━━━━━━┳━━━━━━┳━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━┳━━━━━━━━━┓  │\n",
-       "│ ┃ column         ┃ NA   ┃ NA %   ┃ mean      ┃ sd      ┃ p0    ┃ p25    ┃ p50     ┃ p75   ┃ p100   ┃ hist    ┃  │\n",
-       "│ ┡━━━━━━━━━━━━━━━━╇━━━━━━╇━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━╇━━━━━━━━╇━━━━━━━━━╇━━━━━━━╇━━━━━━━━╇━━━━━━━━━┩  │\n",
-       "│ │ passengers     │    0 │      0 │     1.539 │   1.204 │     0 │      1 │       1 │     2 │      6 │  █▁  ▁  │  │\n",
-       "│ │ distance       │    0 │      0 │     3.025 │   3.828 │     0 │   0.98 │    1.64 │  3.21 │   36.7 │   █▁    │  │\n",
-       "│ │ fare           │    0 │      0 │     13.09 │   11.55 │     1 │    6.5 │     9.5 │    15 │    150 │   █▁    │  │\n",
-       "│ │ tip            │    0 │      0 │     1.979 │   2.449 │     0 │      0 │     1.7 │   2.8 │   33.2 │    █    │  │\n",
-       "│ │ tolls          │    0 │      0 │    0.3253 │   1.415 │     0 │      0 │       0 │     0 │  24.02 │    █    │  │\n",
-       "│ │ total          │    0 │      0 │     18.52 │   13.82 │   1.3 │   10.8 │   14.16 │  20.3 │  174.8 │   █▁    │  │\n",
-       "│ └────────────────┴──────┴────────┴───────────┴─────────┴───────┴────────┴─────────┴───────┴────────┴─────────┘  │\n",
-       "│                                                    category                                                     │\n",
-       "│ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓  │\n",
-       "│ ┃ column                        ┃ NA     ┃ NA %                               ┃ ordered        ┃ unique      ┃  │\n",
-       "│ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩  │\n",
-       "│ │ color                         │      0 │                                  0 │ False          │           2 │  │\n",
-       "│ │ payment                       │     44 │                 0.6839732628633608 │ False          │           3 │  │\n",
-       "│ │ pickup_borough                │     26 │                 0.4041660189647132 │ False          │           5 │  │\n",
-       "│ │ dropoff_borough               │     45 │                  0.699518109746619 │ False          │           6 │  │\n",
-       "│ └───────────────────────────────┴────────┴────────────────────────────────────┴────────────────┴─────────────┘  │\n",
-       "│                                                    datetime                                                     │\n",
-       "│ ┏━━━━━━━━━━━━━┳━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓  │\n",
-       "│ ┃ column      ┃ NA   ┃ NA %    ┃ first                        ┃ last                         ┃ frequency     ┃  │\n",
-       "│ ┡━━━━━━━━━━━━━╇━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩  │\n",
-       "│ │ pickup      │    0 │       0 │     2019-02-28 23:29:03      │     2019-03-31 23:43:45      │ None          │  │\n",
-       "│ │ dropoff     │    0 │       0 │     2019-02-28 23:32:35      │     2019-04-01 00:13:58      │ None          │  │\n",
-       "│ └─────────────┴──────┴─────────┴──────────────────────────────┴──────────────────────────────┴───────────────┘  │\n",
-       "│                                                     string                                                      │\n",
-       "│ ┏━━━━━━━━━━━┳━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┓  │\n",
-       "│ ┃           ┃    ┃           ┃          ┃           ┃          ┃           ┃ chars    ┃ words per ┃ total    ┃  │\n",
-       "│ ┃ column    ┃ NA ┃ NA %      ┃ shortest ┃ longest   ┃ min      ┃ max       ┃ per row  ┃ row       ┃ words    ┃  │\n",
-       "│ ┡━━━━━━━━━━━╇━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━┩  │\n",
-       "│ │ pickup_zo │ 26 │ 0.4041660 │ SoHo     │ Riverdale │ Allerton │ Yorkville │     16.2 │       2.5 │    15791 │  │\n",
-       "│ │ ne        │    │ 189647132 │          │ /North    │ /Pelham  │ West      │          │           │          │  │\n",
-       "│ │           │    │           │          │ Riverdale │ Gardens  │           │          │           │          │  │\n",
-       "│ │           │    │           │          │ /Fieldsto │          │           │          │           │          │  │\n",
-       "│ │           │    │           │          │ n         │          │           │          │           │          │  │\n",
-       "│ │ dropoff_z │ 45 │ 0.6995181 │ SoHo     │ Riverdale │ Allerton │ Yorkville │     16.3 │       2.5 │    15851 │  │\n",
-       "│ │ one       │    │  09746619 │          │ /North    │ /Pelham  │ West      │          │           │          │  │\n",
-       "│ │           │    │           │          │ Riverdale │ Gardens  │           │          │           │          │  │\n",
-       "│ │           │    │           │          │ /Fieldsto │          │           │          │           │          │  │\n",
-       "│ │           │    │           │          │ n         │          │           │          │           │          │  │\n",
-       "│ └───────────┴────┴───────────┴──────────┴───────────┴──────────┴───────────┴──────────┴───────────┴──────────┘  │\n",
-       "╰────────────────────────────────────────────────────── End ──────────────────────────────────────────────────────╯\n",
-       "
\n" - ], - "text/plain": [ - "╭──────────────────────────────────────────────── skimpy summary ─────────────────────────────────────────────────╮\n", - "│ \u001b[3m Data Summary \u001b[0m \u001b[3m Data Types \u001b[0m \u001b[3m Categories \u001b[0m │\n", - "│ ┏━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┓ ┏━━━━━━━━━━━━━┳━━━━━━━┓ ┏━━━━━━━━━━━━━━━━━━━━━━━┓ │\n", - "│ ┃\u001b[1;36m \u001b[0m\u001b[1;36mDataframe \u001b[0m\u001b[1;36m \u001b[0m┃\u001b[1;36m \u001b[0m\u001b[1;36mValues\u001b[0m\u001b[1;36m \u001b[0m┃ ┃\u001b[1;36m \u001b[0m\u001b[1;36mColumn Type\u001b[0m\u001b[1;36m \u001b[0m┃\u001b[1;36m \u001b[0m\u001b[1;36mCount\u001b[0m\u001b[1;36m \u001b[0m┃ ┃\u001b[1;36m \u001b[0m\u001b[1;36mCategorical Variables\u001b[0m\u001b[1;36m \u001b[0m┃ │\n", - "│ ┡━━━━━━━━━━━━━━━━━━━╇━━━━━━━━┩ ┡━━━━━━━━━━━━━╇━━━━━━━┩ ┡━━━━━━━━━━━━━━━━━━━━━━━┩ │\n", - "│ │ Number of rows │ 6433 │ │ float64 │ 5 │ │ color │ │\n", - "│ │ Number of columns │ 14 │ │ category │ 4 │ │ payment │ │\n", - "│ └───────────────────┴────────┘ │ datetime64 │ 2 │ │ pickup_borough │ │\n", - "│ │ string │ 2 │ │ dropoff_borough │ │\n", - "│ │ int64 │ 1 │ └───────────────────────┘ │\n", - "│ └─────────────┴───────┘ │\n", - "│ \u001b[3m number \u001b[0m │\n", - "│ ┏━━━━━━━━━━━━━━━━┳━━━━━━┳━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━┳━━━━━━━━━┓ │\n", - "│ ┃\u001b[1m \u001b[0m\u001b[1mcolumn \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA % \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mmean \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1msd \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mp0 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mp25 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mp50 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mp75 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mp100 \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mhist \u001b[0m\u001b[1m \u001b[0m┃ │\n", - "│ ┡━━━━━━━━━━━━━━━━╇━━━━━━╇━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━╇━━━━━━━━╇━━━━━━━━━╇━━━━━━━╇━━━━━━━━╇━━━━━━━━━┩ │\n", - "│ │ \u001b[38;5;141mpassengers \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 1.539\u001b[0m │ \u001b[36m 1.204\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 1\u001b[0m │ \u001b[36m 1\u001b[0m │ \u001b[36m 2\u001b[0m │ \u001b[36m 6\u001b[0m │ \u001b[32m █▁ ▁ \u001b[0m │ │\n", - "│ │ \u001b[38;5;141mdistance \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 3.025\u001b[0m │ \u001b[36m 3.828\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0.98\u001b[0m │ \u001b[36m 1.64\u001b[0m │ \u001b[36m 3.21\u001b[0m │ \u001b[36m 36.7\u001b[0m │ \u001b[32m █▁ \u001b[0m │ │\n", - "│ │ \u001b[38;5;141mfare \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 13.09\u001b[0m │ \u001b[36m 11.55\u001b[0m │ \u001b[36m 1\u001b[0m │ \u001b[36m 6.5\u001b[0m │ \u001b[36m 9.5\u001b[0m │ \u001b[36m 15\u001b[0m │ \u001b[36m 150\u001b[0m │ \u001b[32m █▁ \u001b[0m │ │\n", - "│ │ \u001b[38;5;141mtip \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 1.979\u001b[0m │ \u001b[36m 2.449\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 1.7\u001b[0m │ \u001b[36m 2.8\u001b[0m │ \u001b[36m 33.2\u001b[0m │ \u001b[32m █ \u001b[0m │ │\n", - "│ │ \u001b[38;5;141mtolls \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0.3253\u001b[0m │ \u001b[36m 1.415\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 24.02\u001b[0m │ \u001b[32m █ \u001b[0m │ │\n", - "│ │ \u001b[38;5;141mtotal \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 18.52\u001b[0m │ \u001b[36m 13.82\u001b[0m │ \u001b[36m 1.3\u001b[0m │ \u001b[36m 10.8\u001b[0m │ \u001b[36m 14.16\u001b[0m │ \u001b[36m 20.3\u001b[0m │ \u001b[36m 174.8\u001b[0m │ \u001b[32m █▁ \u001b[0m │ │\n", - "│ └────────────────┴──────┴────────┴───────────┴─────────┴───────┴────────┴─────────┴───────┴────────┴─────────┘ │\n", - "│ \u001b[3m category \u001b[0m │\n", - "│ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓ │\n", - "│ ┃\u001b[1m \u001b[0m\u001b[1mcolumn \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA % \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mordered \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1munique \u001b[0m\u001b[1m \u001b[0m┃ │\n", - "│ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩ │\n", - "│ │ \u001b[38;5;141mcolor \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[38;5;45mFalse \u001b[0m │ \u001b[36m 2\u001b[0m │ │\n", - "│ │ \u001b[38;5;141mpayment \u001b[0m │ \u001b[36m 44\u001b[0m │ \u001b[36m 0.6839732628633608\u001b[0m │ \u001b[38;5;45mFalse \u001b[0m │ \u001b[36m 3\u001b[0m │ │\n", - "│ │ \u001b[38;5;141mpickup_borough \u001b[0m │ \u001b[36m 26\u001b[0m │ \u001b[36m 0.4041660189647132\u001b[0m │ \u001b[38;5;45mFalse \u001b[0m │ \u001b[36m 5\u001b[0m │ │\n", - "│ │ \u001b[38;5;141mdropoff_borough \u001b[0m │ \u001b[36m 45\u001b[0m │ \u001b[36m 0.699518109746619\u001b[0m │ \u001b[38;5;45mFalse \u001b[0m │ \u001b[36m 6\u001b[0m │ │\n", - "│ └───────────────────────────────┴────────┴────────────────────────────────────┴────────────────┴─────────────┘ │\n", - "│ \u001b[3m datetime \u001b[0m │\n", - "│ ┏━━━━━━━━━━━━━┳━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓ │\n", - "│ ┃\u001b[1m \u001b[0m\u001b[1mcolumn \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA % \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mfirst \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mlast \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mfrequency \u001b[0m\u001b[1m \u001b[0m┃ │\n", - "│ ┡━━━━━━━━━━━━━╇━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩ │\n", - "│ │ \u001b[38;5;141mpickup \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[31m 2019-02-28 23:29:03 \u001b[0m │ \u001b[31m 2019-03-31 23:43:45 \u001b[0m │ \u001b[38;5;141mNone \u001b[0m │ │\n", - "│ │ \u001b[38;5;141mdropoff \u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[36m 0\u001b[0m │ \u001b[31m 2019-02-28 23:32:35 \u001b[0m │ \u001b[31m 2019-04-01 00:13:58 \u001b[0m │ \u001b[38;5;141mNone \u001b[0m │ │\n", - "│ └─────────────┴──────┴─────────┴──────────────────────────────┴──────────────────────────────┴───────────────┘ │\n", - "│ \u001b[3m string \u001b[0m │\n", - "│ ┏━━━━━━━━━━━┳━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┓ │\n", - "│ ┃\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mchars \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mwords per\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mtotal \u001b[0m\u001b[1m \u001b[0m┃ │\n", - "│ ┃\u001b[1m \u001b[0m\u001b[1mcolumn \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mNA % \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mshortest\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mlongest \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mmin \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mmax \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mper row \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mrow \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mwords \u001b[0m\u001b[1m \u001b[0m┃ │\n", - "│ ┡━━━━━━━━━━━╇━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━┩ │\n", - "│ │ \u001b[38;5;141mpickup_zo\u001b[0m │ \u001b[36m26\u001b[0m │ \u001b[36m0.4041660\u001b[0m │ \u001b[38;5;141mSoHo \u001b[0m │ \u001b[38;5;141mRiverdale\u001b[0m │ \u001b[38;5;141mAllerton\u001b[0m │ \u001b[38;5;141mYorkville\u001b[0m │ \u001b[36m 16.2\u001b[0m │ \u001b[36m 2.5\u001b[0m │ \u001b[36m 15791\u001b[0m │ │\n", - "│ │ \u001b[38;5;141mne \u001b[0m │ │ \u001b[36m189647132\u001b[0m │ │ \u001b[38;5;141m/North \u001b[0m │ \u001b[38;5;141m/Pelham \u001b[0m │ \u001b[38;5;141mWest \u001b[0m │ │ │ │ │\n", - "│ │ │ │ │ │ \u001b[38;5;141mRiverdale\u001b[0m │ \u001b[38;5;141mGardens \u001b[0m │ │ │ │ │ │\n", - "│ │ │ │ │ │ \u001b[38;5;141m/Fieldsto\u001b[0m │ │ │ │ │ │ │\n", - "│ │ │ │ │ │ \u001b[38;5;141mn \u001b[0m │ │ │ │ │ │ │\n", - "│ │ \u001b[38;5;141mdropoff_z\u001b[0m │ \u001b[36m45\u001b[0m │ \u001b[36m0.6995181\u001b[0m │ \u001b[38;5;141mSoHo \u001b[0m │ \u001b[38;5;141mRiverdale\u001b[0m │ \u001b[38;5;141mAllerton\u001b[0m │ \u001b[38;5;141mYorkville\u001b[0m │ \u001b[36m 16.3\u001b[0m │ \u001b[36m 2.5\u001b[0m │ \u001b[36m 15851\u001b[0m │ │\n", - "│ │ \u001b[38;5;141mone \u001b[0m │ │ \u001b[36m 09746619\u001b[0m │ │ \u001b[38;5;141m/North \u001b[0m │ \u001b[38;5;141m/Pelham \u001b[0m │ \u001b[38;5;141mWest \u001b[0m │ │ │ │ │\n", - "│ │ │ │ │ │ \u001b[38;5;141mRiverdale\u001b[0m │ \u001b[38;5;141mGardens \u001b[0m │ │ │ │ │ │\n", - "│ │ │ │ │ │ \u001b[38;5;141m/Fieldsto\u001b[0m │ │ │ │ │ │ │\n", - "│ │ │ │ │ │ \u001b[38;5;141mn \u001b[0m │ │ │ │ │ │ │\n", - "│ └───────────┴────┴───────────┴──────────┴───────────┴──────────┴───────────┴──────────┴───────────┴──────────┘ │\n", - "╰────────────────────────────────────────────────────── End ──────────────────────────────────────────────────────╯\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "skim(taxis)" ] diff --git a/functions.ipynb b/functions.ipynb index 0f2cce3..5ed6b2a 100644 --- a/functions.ipynb +++ b/functions.ipynb @@ -48,18 +48,10 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "0450ad6c", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Hello Ada, and welcome!\n" - ] - } - ], + "outputs": [], "source": [ "def welcome_message(name):\n", " return f\"Hello {name}, and welcome!\"\n", @@ -81,19 +73,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "dd49bee5", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "('Hello student', 4.8)\n", - "('Hello Ada', 4.8)\n" - ] - } - ], + "outputs": [], "source": [ "def score_message(score, name=\"student\"):\n", " \"\"\"This is a doc-string, a string describing a function.\n", @@ -138,28 +121,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "50dfff24", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Help on function score_message in module __main__:\n", - "\n", - "score_message(score, name='student')\n", - " This is a doc-string, a string describing a function.\n", - " Args:\n", - " score (float): Raw score\n", - " name (str): Name of student\n", - " Returns:\n", - " str: A hello message.\n", - " float: A normalised score.\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "help(score_message)" ] @@ -194,47 +159,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "43d10e40", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (10, 4)
abcd
f64f64f64f64
0.4736410.3145550.780770.683499
1.00.1921670.8502790.186199
0.4507660.4572280.5157880.0
0.3617790.2195290.2433410.934585
0.028510.00.00.948462
0.4747280.1190681.00.556861
0.1679770.6302890.6544850.727404
0.279770.5057040.9460620.527484
0.4111420.8182760.1750241.0
0.00.1704820.3733090.45716
" - ], - "text/plain": [ - "shape: (10, 4)\n", - "┌──────────┬──────────┬──────────┬──────────┐\n", - "│ a ┆ b ┆ c ┆ d │\n", - "│ --- ┆ --- ┆ --- ┆ --- │\n", - "│ f64 ┆ f64 ┆ f64 ┆ f64 │\n", - "╞══════════╪══════════╪══════════╪══════════╡\n", - "│ 0.473641 ┆ 0.314555 ┆ 0.78077 ┆ 0.683499 │\n", - "│ 1.0 ┆ 0.192167 ┆ 0.850279 ┆ 0.186199 │\n", - "│ 0.450766 ┆ 0.457228 ┆ 0.515788 ┆ 0.0 │\n", - "│ 0.361779 ┆ 0.219529 ┆ 0.243341 ┆ 0.934585 │\n", - "│ 0.02851 ┆ 0.0 ┆ 0.0 ┆ 0.948462 │\n", - "│ 0.474728 ┆ 0.119068 ┆ 1.0 ┆ 0.556861 │\n", - "│ 0.167977 ┆ 0.630289 ┆ 0.654485 ┆ 0.727404 │\n", - "│ 0.27977 ┆ 0.505704 ┆ 0.946062 ┆ 0.527484 │\n", - "│ 0.411142 ┆ 0.818276 ┆ 0.175024 ┆ 1.0 │\n", - "│ 0.0 ┆ 0.170482 ┆ 0.373309 ┆ 0.45716 │\n", - "└──────────┴──────────┴──────────┴──────────┘" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import numpy as np\n", "import polars as pl\n", @@ -313,7 +241,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "406648b6", "metadata": {}, "outputs": [], @@ -347,76 +275,20 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "74834d90", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (3,)
f64
0.0
0.5
1.0
" - ], - "text/plain": [ - "shape: (3,)\n", - "Series: '' [f64]\n", - "[\n", - "\t0.0\n", - "\t0.5\n", - "\t1.0\n", - "]" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "rescale(pl.Series([-10, 0, 10]))" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "8a0e643e", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (5,)
f64
0.0
0.25
0.5
null
1.0
" - ], - "text/plain": [ - "shape: (5,)\n", - "Series: '' [f64]\n", - "[\n", - "\t0.0\n", - "\t0.25\n", - "\t0.5\n", - "\tnull\n", - "\t1.0\n", - "]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "rescale(pl.Series([1, 2, 3, None, 5]))" ] @@ -434,7 +306,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "ce066e4f", "metadata": {}, "outputs": [], @@ -462,39 +334,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "f8b80894", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (5,)
f64
0.0
0.0
0.0
NaN
0.0
" - ], - "text/plain": [ - "shape: (5,)\n", - "Series: '' [f64]\n", - "[\n", - "\t0.0\n", - "\t0.0\n", - "\t0.0\n", - "\tNaN\n", - "\t0.0\n", - "]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "rescale(pl.Series([1, 2, 3, float(\"inf\"), 5], strict=False))" ] @@ -509,39 +352,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "cd63ddea", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (5,)
f64
0.0
0.25
0.5
null
1.0
" - ], - "text/plain": [ - "shape: (5,)\n", - "Series: '' [f64]\n", - "[\n", - "\t0.0\n", - "\t0.25\n", - "\t0.5\n", - "\tnull\n", - "\t1.0\n", - "]" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "def rescale(x):\n", " x = x.cast(pl.Float64).replace(float(\"inf\"), None)\n", @@ -627,18 +441,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "647da443", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Hello World!\n" - ] - } - ], + "outputs": [], "source": [ "def var_func():\n", " str_variable = \"Hello World!\"\n", @@ -661,19 +467,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "c5fb049d", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "y is inside a function: I'm a global variable\n", - "y is outside a function: I'm a global variable\n" - ] - } - ], + "outputs": [], "source": [ "y = \"I'm a global variable\"\n", "\n", diff --git a/introduction.ipynb b/introduction.ipynb index 2a70fd3..04c2ff9 100644 --- a/introduction.ipynb +++ b/introduction.ipynb @@ -17,7 +17,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "51a55374", "metadata": { "tags": [ @@ -33,116 +33,14 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "209ef434", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "cluster_0\n", - "\n", - "Understand\n", - "\n", - "\n", - "\n", - "Import\n", - "\n", - "Import\n", - "\n", - "\n", - "\n", - "Clean\n", - "\n", - "Clean\n", - "\n", - "\n", - "\n", - "Import->Clean\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Analyse\n", - "\n", - "Analyse\n", - "\n", - "\n", - "\n", - "Clean->Analyse\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Visualise\n", - "\n", - "Visualise\n", - "\n", - "\n", - "\n", - "Visualise->Analyse\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Transform\n", - "\n", - "Transform\n", - "\n", - "\n", - "\n", - "Analyse->Transform\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Communicate\n", - "\n", - "Communicate\n", - "\n", - "\n", - "\n", - "Analyse->Communicate\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Transform->Visualise\n", - "\n", - "\n", - "\n", - "\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# | echo: false\n", "import graphviz\n", @@ -240,22 +138,14 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "26faf349", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Compiled with Python version: 3.12.13 (main, Mar 3 2026, 12:39:30) [Clang 17.0.0 (clang-1700.6.3.2)]\n" - ] - } - ], + "outputs": [], "source": [ "# | echo: false\n", "import sys\n", diff --git a/iteration.ipynb b/iteration.ipynb index fd727cd..b44f79a 100644 --- a/iteration.ipynb +++ b/iteration.ipynb @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "51a55374", "metadata": { "tags": [ @@ -68,21 +68,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "a2bbd41c", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Lovelace\n", - "Smith\n", - "Pigou\n", - "Babbage\n" - ] - } - ], + "outputs": [], "source": [ "name_list = [\"Lovelace\", \"Smith\", \"Pigou\", \"Babbage\"]\n", "\n", @@ -117,19 +106,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "835ebda7", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hello\n", - "---end entry---\n" - ] - } - ], + "outputs": [], "source": [ "for entry in [\"hello\"]:\n", " print(entry)\n", @@ -146,27 +126,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "2a19ac2e", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "h\n", - "---end entry---\n", - "e\n", - "---end entry---\n", - "l\n", - "---end entry---\n", - "l\n", - "---end entry---\n", - "o\n", - "---end entry---\n" - ] - } - ], + "outputs": [], "source": [ "for entry in \"hello\":\n", " print(entry)\n", @@ -187,21 +150,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "239e133f", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The name in position 0 is Lovelace\n", - "The name in position 1 is Smith\n", - "The name in position 2 is Hopper\n", - "The name in position 3 is Babbage\n" - ] - } - ], + "outputs": [], "source": [ "name_list = [\"Lovelace\", \"Smith\", \"Hopper\", \"Babbage\"]\n", "\n", @@ -219,21 +171,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "b66c5c53", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The name in position 1 is Lovelace\n", - "The name in position 2 is Smith\n", - "The name in position 3 is Hopper\n", - "The name in position 4 is Babbage\n" - ] - } - ], + "outputs": [], "source": [ "for i, name in enumerate(name_list, start=1):\n", " print(f\"The name in position {i} is {name}\")" @@ -249,21 +190,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "010239fe", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "In Paris, the temperature is 28 degrees C today.\n", - "In London, the temperature is 22 degrees C today.\n", - "In Seville, the temperature is 36 degrees C today.\n", - "In Wellesley, the temperature is 29 degrees C today.\n" - ] - } - ], + "outputs": [], "source": [ "cities_to_temps = {\"Paris\": 28, \"London\": 22, \"Seville\": 36, \"Wellesley\": 29}\n", "\n", @@ -287,21 +217,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "8ea3efc5", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Ada Lovelace\n", - "Adam Smith\n", - "Grace Hopper\n", - "Charles Babbage\n" - ] - } - ], + "outputs": [], "source": [ "first_names = [\"Ada\", \"Adam\", \"Grace\", \"Charles\"]\n", "last_names = [\"Lovelace\", \"Smith\", \"Hopper\", \"Babbage\"]\n", @@ -338,21 +257,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "7efed381", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[51, 52, 53, 54, 55, 56, 57, 58, 59, 60]" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "num_list = range(50, 60)\n", "[1 + num for num in num_list]" @@ -378,18 +286,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "722fda21", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39]\n" - ] - } - ], + "outputs": [], "source": [ "number_list = range(1, 40)\n", "divide_list = [x for x in number_list if x % 3 == 0]\n", @@ -408,18 +308,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "b6e80d6b", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['Adam Smith', 'leonara smith']\n" - ] - } - ], + "outputs": [], "source": [ "names_list = [\"Joe Bloggs\", \"Adam Smith\", \"Sandra Noone\", \"leonara smith\"]\n", "smith_list = [x for x in names_list if \"smith\" in x.lower()]\n", @@ -438,18 +330,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "f348bfb6", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['Not Smith!', 'Adam Smith', 'Not Smith!', 'leonara smith']\n" - ] - } - ], + "outputs": [], "source": [ "names_list = [\"Joe Bloggs\", \"Adam Smith\", \"Sandra Noone\", \"leonara smith\"]\n", "smith_list = [x if \"smith\" in x.lower() else \"Not Smith!\" for x in names_list]\n", @@ -466,18 +350,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "74e4fcc7", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['Ada Lovelace', 'Adam Smith', 'Grace Hopper', 'Charles Babbage']\n" - ] - } - ], + "outputs": [], "source": [ "first_names = [\"Ada\", \"Adam\", \"Grace\", \"Charles\"]\n", "last_names = [\"Lovelace\", \"Smith\", \"Hopper\", \"Babbage\"]\n", @@ -495,18 +371,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "2c82cf1f", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[['Ada Lovelace', 'Adam Lovelace'], ['Ada Smith', 'Adam Smith']]\n" - ] - } - ], + "outputs": [], "source": [ "first_names = [\"Ada\", \"Adam\"]\n", "last_names = [\"Lovelace\", \"Smith\"]\n", @@ -526,21 +394,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "acef16ae", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Ada': 'Lovelace', 'Adam': 'Smith'}" - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "{key: value for key, value in zip(first_names, last_names)}" ] @@ -563,28 +420,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "3e47ba02", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10\n", - "9\n", - "8\n", - "7\n", - "6\n", - "5\n", - "4\n", - "3\n", - "2\n", - "1\n", - "execution complete\n" - ] - } - ], + "outputs": [], "source": [ "n = 10\n", "while n > 0:\n", @@ -637,43 +476,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "b3116809", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 4)
abcd
f64f64f64f64
-0.871189-0.681725-1.562824-0.560553
-1.2694980.148269-0.217629-0.221637
-0.555918-1.4479222.4705161.641996
0.35463-1.664464-0.0148610.030461
-0.8667982.2567060.968797-0.078444
0.5406780.8744340.286314-0.138539
" - ], - "text/plain": [ - "shape: (6, 4)\n", - "┌───────────┬───────────┬───────────┬───────────┐\n", - "│ a ┆ b ┆ c ┆ d │\n", - "│ --- ┆ --- ┆ --- ┆ --- │\n", - "│ f64 ┆ f64 ┆ f64 ┆ f64 │\n", - "╞═══════════╪═══════════╪═══════════╪═══════════╡\n", - "│ -0.871189 ┆ -0.681725 ┆ -1.562824 ┆ -0.560553 │\n", - "│ -1.269498 ┆ 0.148269 ┆ -0.217629 ┆ -0.221637 │\n", - "│ -0.555918 ┆ -1.447922 ┆ 2.470516 ┆ 1.641996 │\n", - "│ 0.35463 ┆ -1.664464 ┆ -0.014861 ┆ 0.030461 │\n", - "│ -0.866798 ┆ 2.256706 ┆ 0.968797 ┆ -0.078444 │\n", - "│ 0.540678 ┆ 0.874434 ┆ 0.286314 ┆ -0.138539 │\n", - "└───────────┴───────────┴───────────┴───────────┘" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import numpy as np\n", "import polars as pl\n", @@ -692,81 +498,20 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "ac909c2f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (1, 4)
abcd
f64f64f64f64
-0.711358-0.2667280.135727-0.108492
" - ], - "text/plain": [ - "shape: (1, 4)\n", - "┌───────────┬───────────┬──────────┬───────────┐\n", - "│ a ┆ b ┆ c ┆ d │\n", - "│ --- ┆ --- ┆ --- ┆ --- │\n", - "│ f64 ┆ f64 ┆ f64 ┆ f64 │\n", - "╞═══════════╪═══════════╪══════════╪═══════════╡\n", - "│ -0.711358 ┆ -0.266728 ┆ 0.135727 ┆ -0.108492 │\n", - "└───────────┴───────────┴──────────┴───────────┘" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.select(pl.all().median())" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "96426002", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 1)
row_median
f64
-0.776457
-0.219633
0.543039
0.0078
0.445177
0.413496
" - ], - "text/plain": [ - "shape: (6, 1)\n", - "┌────────────┐\n", - "│ row_median │\n", - "│ --- │\n", - "│ f64 │\n", - "╞════════════╡\n", - "│ -0.776457 │\n", - "│ -0.219633 │\n", - "│ 0.543039 │\n", - "│ 0.0078 │\n", - "│ 0.445177 │\n", - "│ 0.413496 │\n", - "└────────────┘" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.select(pl.concat_list(pl.all()).list.median().alias(\"row_median\"))" ] @@ -781,18 +526,10 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "060b6815", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "294 μs ± 4.83 μs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)\n" - ] - } - ], + "outputs": [], "source": [ "# Do not do this!\n", "\n", @@ -816,18 +553,10 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "1a48ae52", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "56.5 μs ± 929 ns per loop (mean ± std. dev. of 7 runs, 10,000 loops each)\n" - ] - } - ], + "outputs": [], "source": [ "%timeit df + 5" ] @@ -852,7 +581,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "f7391dc5", "metadata": {}, "outputs": [], @@ -874,43 +603,10 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "31adcb3f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 6)
abcdnew_aresult
f64f64f64f64f64f64
40554.12881140554.31827540553.43717640554.43944740559.128811-4.545293
40553.73050240555.14826940554.78237140554.77836340558.730502-5.458877
40554.44408240553.55207840557.47051640556.64199640559.444082-9.030254
40555.3546340553.33553640554.98513940555.03046140560.35463-5.850543
40554.13320240557.25670640555.96879740554.92155640559.133202-4.134134
40555.54067840555.87443440555.28631440554.86146140560.540678-3.426466
" - ], - "text/plain": [ - "shape: (6, 6)\n", - "┌──────────────┬──────────────┬──────────────┬──────────────┬──────────────┬───────────┐\n", - "│ a ┆ b ┆ c ┆ d ┆ new_a ┆ result │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │\n", - "╞══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪═══════════╡\n", - "│ 40554.128811 ┆ 40554.318275 ┆ 40553.437176 ┆ 40554.439447 ┆ 40559.128811 ┆ -4.545293 │\n", - "│ 40553.730502 ┆ 40555.148269 ┆ 40554.782371 ┆ 40554.778363 ┆ 40558.730502 ┆ -5.458877 │\n", - "│ 40554.444082 ┆ 40553.552078 ┆ 40557.470516 ┆ 40556.641996 ┆ 40559.444082 ┆ -9.030254 │\n", - "│ 40555.35463 ┆ 40553.335536 ┆ 40554.985139 ┆ 40555.030461 ┆ 40560.35463 ┆ -5.850543 │\n", - "│ 40554.133202 ┆ 40557.256706 ┆ 40555.968797 ┆ 40554.921556 ┆ 40559.133202 ┆ -4.134134 │\n", - "│ 40555.540678 ┆ 40555.874434 ┆ 40555.286314 ┆ 40554.861461 ┆ 40560.540678 ┆ -3.426466 │\n", - "└──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴───────────┘" - ] - }, - "execution_count": 49, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Don't do this (slow, row-wise)\n", "mean_new_a = df.select(pl.col(\"new_a\").mean()).item()\n", @@ -942,43 +638,10 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "8d9defd3", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 6)
abcdnew_aratio
f64f64f64f64f64f64
40554.12881140554.31827540553.43717640554.43944740559.1288110.999877
40553.73050240555.14826940554.78237140554.77836340558.7305020.999877
40554.44408240553.55207840557.47051640556.64199640559.4440820.999877
40555.3546340553.33553640554.98513940555.03046140560.354630.999877
40554.13320240557.25670640555.96879740554.92155640559.1332020.999877
40555.54067840555.87443440555.28631440554.86146140560.5406780.999877
" - ], - "text/plain": [ - "shape: (6, 6)\n", - "┌──────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────┐\n", - "│ a ┆ b ┆ c ┆ d ┆ new_a ┆ ratio │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │\n", - "╞══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════╡\n", - "│ 40554.128811 ┆ 40554.318275 ┆ 40553.437176 ┆ 40554.439447 ┆ 40559.128811 ┆ 0.999877 │\n", - "│ 40553.730502 ┆ 40555.148269 ┆ 40554.782371 ┆ 40554.778363 ┆ 40558.730502 ┆ 0.999877 │\n", - "│ 40554.444082 ┆ 40553.552078 ┆ 40557.470516 ┆ 40556.641996 ┆ 40559.444082 ┆ 0.999877 │\n", - "│ 40555.35463 ┆ 40553.335536 ┆ 40554.985139 ┆ 40555.030461 ┆ 40560.35463 ┆ 0.999877 │\n", - "│ 40554.133202 ┆ 40557.256706 ┆ 40555.968797 ┆ 40554.921556 ┆ 40559.133202 ┆ 0.999877 │\n", - "│ 40555.540678 ┆ 40555.874434 ┆ 40555.286314 ┆ 40554.861461 ┆ 40560.540678 ┆ 0.999877 │\n", - "└──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────┘" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = df.with_columns((pl.col(\"a\") / pl.col(\"new_a\")).alias(\"ratio\"))\n", "df" @@ -994,43 +657,10 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "f301c8cb", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 7)
abcdnew_aratioa_gt_0.5
f64f64f64f64f64f64bool
40554.12881140554.31827540553.43717640554.43944740559.1288110.999877true
40553.73050240555.14826940554.78237140554.77836340558.7305020.999877true
40554.44408240553.55207840557.47051640556.64199640559.4440820.999877true
40555.3546340553.33553640554.98513940555.03046140560.354630.999877true
40554.13320240557.25670640555.96879740554.92155640559.1332020.999877true
40555.54067840555.87443440555.28631440554.86146140560.5406780.999877true
" - ], - "text/plain": [ - "shape: (6, 7)\n", - "┌──────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────┬──────────┐\n", - "│ a ┆ b ┆ c ┆ d ┆ new_a ┆ ratio ┆ a_gt_0.5 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ bool │\n", - "╞══════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════╪══════════╡\n", - "│ 40554.128811 ┆ 40554.318275 ┆ 40553.437176 ┆ 40554.439447 ┆ 40559.128811 ┆ 0.999877 ┆ true │\n", - "│ 40553.730502 ┆ 40555.148269 ┆ 40554.782371 ┆ 40554.778363 ┆ 40558.730502 ┆ 0.999877 ┆ true │\n", - "│ 40554.444082 ┆ 40553.552078 ┆ 40557.470516 ┆ 40556.641996 ┆ 40559.444082 ┆ 0.999877 ┆ true │\n", - "│ 40555.35463 ┆ 40553.335536 ┆ 40554.985139 ┆ 40555.030461 ┆ 40560.35463 ┆ 0.999877 ┆ true │\n", - "│ 40554.133202 ┆ 40557.256706 ┆ 40555.968797 ┆ 40554.921556 ┆ 40559.133202 ┆ 0.999877 ┆ true │\n", - "│ 40555.540678 ┆ 40555.874434 ┆ 40555.286314 ┆ 40554.861461 ┆ 40560.540678 ┆ 0.999877 ┆ true │\n", - "└──────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────┴──────────┘" - ] - }, - "execution_count": 51, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = df.with_columns((pl.col(\"a\") > 0.5).alias(\"a_gt_0.5\"))\n", "df" diff --git a/joins.ipynb b/joins.ipynb index a1b57a3..d494d84 100644 --- a/joins.ipynb +++ b/joins.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "51a55374", "metadata": { "tags": [ @@ -64,98 +64,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "f5ef4f37", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " county pop\n", - "0 Los Angeles 9878554\n", - "1 Orange 2997033\n", - "2 Ventura 798364\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
countypop
ca0Los Angeles9878554
1Orange2997033
2Ventura798364
il0Cook5285107
1DeKalb103729
2Will673586
\n", - "
" - ], - "text/plain": [ - " county pop\n", - "ca 0 Los Angeles 9878554\n", - " 1 Orange 2997033\n", - " 2 Ventura 798364\n", - "il 0 Cook 5285107\n", - " 1 DeKalb 103729\n", - " 2 Will 673586" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import pandas as pd\n", "\n", @@ -218,93 +130,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "53c66d5d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
key1key2ABCD
0K0K0A0B0C0D0
1K1K0A2B2C1D1
2K1K0A2B2C2D2
3K2K0NaNNaNC3D3
\n", - "
" - ], - "text/plain": [ - " key1 key2 A B C D\n", - "0 K0 K0 A0 B0 C0 D0\n", - "1 K1 K0 A2 B2 C1 D1\n", - "2 K1 K0 A2 B2 C2 D2\n", - "3 K2 K0 NaN NaN C3 D3" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "left = pd.DataFrame(\n", " {\n", @@ -338,83 +167,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "5e73608f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
key1key2ABCD
0K0K0A0B0C0D0
1K1K0A2B2C1D1
2K1K0A2B2C2D2
\n", - "
" - ], - "text/plain": [ - " key1 key2 A B C D\n", - "0 K0 K0 A0 B0 C0 D0\n", - "1 K1 K0 A2 B2 C1 D1\n", - "2 K1 K0 A2 B2 C2 D2" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pd.merge(left, right, on=[\"key1\", \"key2\"], how=\"inner\")" ] @@ -431,120 +187,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "5d209fb9", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
key1key2ABCD_merge
0K0K0A0B0C0D0both
1K0K1A1B1NaNNaNleft_only
2K1K0A2B2C1D1both
3K1K0A2B2C2D2both
4K2K0NaNNaNC3D3right_only
5K2K1A3B3NaNNaNleft_only
\n", - "
" - ], - "text/plain": [ - " key1 key2 A B C D _merge\n", - "0 K0 K0 A0 B0 C0 D0 both\n", - "1 K0 K1 A1 B1 NaN NaN left_only\n", - "2 K1 K0 A2 B2 C1 D1 both\n", - "3 K1 K0 A2 B2 C2 D2 both\n", - "4 K2 K0 NaN NaN C3 D3 right_only\n", - "5 K2 K1 A3 B3 NaN NaN left_only" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pd.merge(left, right, on=[\"key1\", \"key2\"], how=\"outer\", indicator=True)" ] diff --git a/missing-values.ipynb b/missing-values.ipynb index 08aed7c..67e95ef 100644 --- a/missing-values.ipynb +++ b/missing-values.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "51a55374", "metadata": { "tags": [ @@ -61,68 +61,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "535ef959", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
numbers
05.0
127.3
2NaN
3-16.0
\n", - "
" - ], - "text/plain": [ - " numbers\n", - "0 5.0\n", - "1 27.3\n", - "2 NaN\n", - "3 -16.0" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", @@ -141,73 +83,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "bf317bce", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
numbers
0<NA>
127.3
2NaN
3-16
4None
\n", - "
" - ], - "text/plain": [ - " numbers\n", - "0 \n", - "1 27.3\n", - "2 NaN\n", - "3 -16\n", - "4 None" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "numbers = pd.DataFrame([pd.NA, 27.3, np.nan, -16, None], columns=[\"numbers\"])\n", "numbers" @@ -223,78 +102,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "b1c864d9", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
fruit
0orange
1NaN
2apple
3None
4banana
5<NA>
\n", - "
" - ], - "text/plain": [ - " fruit\n", - "0 orange\n", - "1 NaN\n", - "2 apple\n", - "3 None\n", - "4 banana\n", - "5 " - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "fruits = pd.DataFrame(\n", " [\"orange\", np.nan, \"apple\", None, \"banana\", pd.NA], columns=[\"fruit\"]\n", @@ -312,78 +123,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "cbcfe630", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
fruit
0False
1True
2False
3True
4False
5True
\n", - "
" - ], - "text/plain": [ - " fruit\n", - "0 False\n", - "1 True\n", - "2 False\n", - "3 True\n", - "4 False\n", - "5 True" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "fruits.isna()" ] @@ -408,83 +151,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "c96f89e6", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ABCD
0NaN2.0NaN0
13.04.0NaN1
25.0NaNNaN<NA>
3NaN3.0NaN4
\n", - "
" - ], - "text/plain": [ - " A B C D\n", - "0 NaN 2.0 NaN 0\n", - "1 3.0 4.0 NaN 1\n", - "2 5.0 NaN NaN \n", - "3 NaN 3.0 NaN 4" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "nan_df = pd.DataFrame(\n", " [\n", @@ -509,91 +179,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "a6bc5fe2", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/4054961691.py:1: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", - " nan_df.fillna(0)\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ABCD
00.02.00.00
13.04.00.01
25.00.00.00
30.03.00.04
\n", - "
" - ], - "text/plain": [ - " A B C D\n", - "0 0.0 2.0 0.0 0\n", - "1 3.0 4.0 0.0 1\n", - "2 5.0 0.0 0.0 0\n", - "3 0.0 3.0 0.0 4" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "nan_df.fillna(0)" ] @@ -608,91 +197,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "e65b67c2", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/2397886090.py:1: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", - " nan_df.fillna(value={\"A\": 0, \"B\": 1, \"C\": 2, \"D\": 3})\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ABCD
00.02.02.00
13.04.02.01
25.01.02.03
30.03.02.04
\n", - "
" - ], - "text/plain": [ - " A B C D\n", - "0 0.0 2.0 2.0 0\n", - "1 3.0 4.0 2.0 1\n", - "2 5.0 1.0 2.0 3\n", - "3 0.0 3.0 2.0 4" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "nan_df.fillna(value={\"A\": 0, \"B\": 1, \"C\": 2, \"D\": 3})" ] @@ -707,186 +215,20 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "2a19e196", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/1353804149.py:1: FutureWarning: DataFrame.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.\n", - " nan_df.fillna(method=\"ffill\")\n", - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/1353804149.py:1: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", - " nan_df.fillna(method=\"ffill\")\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ABCD
0NaN2.0NaN0
13.04.0NaN1
25.04.0NaN1
35.03.0NaN4
\n", - "
" - ], - "text/plain": [ - " A B C D\n", - "0 NaN 2.0 NaN 0\n", - "1 3.0 4.0 NaN 1\n", - "2 5.0 4.0 NaN 1\n", - "3 5.0 3.0 NaN 4" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "nan_df.fillna(method=\"ffill\")" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "8b5b001e", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/2505504399.py:1: FutureWarning: DataFrame.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.\n", - " nan_df.fillna(method=\"bfill\")\n", - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/2505504399.py:1: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", - " nan_df.fillna(method=\"bfill\")\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ABCD
03.02.0NaN0
13.04.0NaN1
25.03.0NaN4
3NaN3.0NaN4
\n", - "
" - ], - "text/plain": [ - " A B C D\n", - "0 3.0 2.0 NaN 0\n", - "1 3.0 4.0 NaN 1\n", - "2 5.0 3.0 NaN 4\n", - "3 NaN 3.0 NaN 4" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "nan_df.fillna(method=\"bfill\")" ] @@ -909,91 +251,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "558e7a23", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/1730877720.py:1: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", - " nan_df.fillna(value={\"A\": 0, \"B\": 1, \"C\": 2, \"D\": 3}, limit=1)\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ABCD
00.02.02.00
13.04.0NaN1
25.01.0NaN3
3NaN3.0NaN4
\n", - "
" - ], - "text/plain": [ - " A B C D\n", - "0 0.0 2.0 2.0 0\n", - "1 3.0 4.0 NaN 1\n", - "2 5.0 1.0 NaN 3\n", - "3 NaN 3.0 NaN 4" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "nan_df.fillna(value={\"A\": 0, \"B\": 1, \"C\": 2, \"D\": 3}, limit=1)" ] @@ -1008,84 +269,20 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "0e3a81c1", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1 3.0\n", - "2 5.0\n", - "Name: A, dtype: float64" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "nan_df[\"A\"].dropna(axis=0) # on a single column" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "9c1e312e", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
0
1
2
3
\n", - "
" - ], - "text/plain": [ - "Empty DataFrame\n", - "Columns: []\n", - "Index: [0, 1, 2, 3]" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "nan_df.dropna(axis=1)" ] @@ -1100,83 +297,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "3296ea35", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ABCD
0NaN2.0NaN0
13.04.0NaN1
25.0NaNNaN<NA>
3NaN3.0NaN4
\n", - "
" - ], - "text/plain": [ - " A B C D\n", - "0 NaN 2.0 NaN 0\n", - "1 3.0 4.0 NaN 1\n", - "2 5.0 NaN NaN \n", - "3 NaN 3.0 NaN 4" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "nan_df.dropna(how=\"all\")" ] @@ -1199,67 +323,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "d3e1af81", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ABCD
13.04.0NaN1
25.0NaNNaN<NA>
\n", - "
" - ], - "text/plain": [ - " A B C D\n", - "1 3.0 4.0 NaN 1\n", - "2 5.0 NaN NaN " - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "nan_df[nan_df[\"A\"].notna()]" ] @@ -1278,71 +345,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "45668d12", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ABC
0345
1-74-99
2-9965
\n", - "
" - ], - "text/plain": [ - " A B C\n", - "0 3 4 5\n", - "1 -7 4 -99\n", - "2 -99 6 5" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "stata_df = pd.DataFrame([[3, 4, 5], [-7, 4, -99], [-99, 6, 5]], columns=list(\"ABC\"))\n", "\n", @@ -1359,71 +365,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "c43070f4", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ABC
0345
1-74<NA>
2<NA>65
\n", - "
" - ], - "text/plain": [ - " A B C\n", - "0 3 4 5\n", - "1 -7 4 \n", - "2 6 5" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "stata_df.replace({-99: pd.NA})" ] @@ -1438,71 +383,10 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "12cc567a", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ABC
0345
1<NA>4<NA>
2<NA>65
\n", - "
" - ], - "text/plain": [ - " A B C\n", - "0 3 4 5\n", - "1 4 \n", - "2 6 5" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "stata_df.replace({-99: pd.NA, -7: pd.NA})" ] @@ -1529,99 +413,10 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "057c5203", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
yearqtrprice
0202011.88
1202020.59
2202030.35
320204NaN
4202120.92
5202130.17
6202142.66
\n", - "
" - ], - "text/plain": [ - " year qtr price\n", - "0 2020 1 1.88\n", - "1 2020 2 0.59\n", - "2 2020 3 0.35\n", - "3 2020 4 NaN\n", - "4 2021 2 0.92\n", - "5 2021 3 0.17\n", - "6 2021 4 2.66" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "stocks = pd.DataFrame(\n", " {\n", @@ -1667,75 +462,10 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "87ab37d0", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
qtr1234
year
20201.880.590.35NaN
2021NaN0.920.172.66
\n", - "
" - ], - "text/plain": [ - "qtr 1 2 3 4\n", - "year \n", - "2020 1.88 0.59 0.35 NaN\n", - "2021 NaN 0.92 0.17 2.66" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "stocks.pivot(columns=\"qtr\", values=\"price\", index=\"year\")" ] @@ -1762,7 +492,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "92a914e3", "metadata": {}, "outputs": [], @@ -1787,78 +517,10 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "cc904247", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
namesmokerage
0Ikaiano34
1Olettano88
2Leriahpreviously75
3Dashayno47
\n", - "
" - ], - "text/plain": [ - " name smoker age\n", - "0 Ikaia no 34\n", - "1 Oletta no 88\n", - "2 Leriah previously 75\n", - "3 Dashay no 47" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "health_cut = health.iloc[:-1, :]\n", "health_cut" @@ -1874,25 +536,10 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "546bc395", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "smoker\n", - "no 3\n", - "previously 1\n", - "yes 0\n", - "Name: count, dtype: int64" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "health_cut[\"smoker\"].value_counts()" ] @@ -1907,33 +554,10 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "efb6c6ab", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61211/3998383890.py:1: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n", - " health_cut.groupby(\"smoker\")[\"age\"].mean()\n" - ] - }, - { - "data": { - "text/plain": [ - "smoker\n", - "no 56.333333\n", - "previously 75.000000\n", - "yes NaN\n", - "Name: age, dtype: float64" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "health_cut.groupby(\"smoker\")[\"age\"].mean()" ] diff --git a/numbers.ipynb b/numbers.ipynb index 129d361..0b96fbb 100644 --- a/numbers.ipynb +++ b/numbers.ipynb @@ -20,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "51a55374", "metadata": { "tags": [ @@ -39,7 +39,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "6c89ca3d", "metadata": {}, "outputs": [], @@ -62,21 +62,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "18f1ee4c", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "np.int64(336776)" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights[\"dest\"].count()" ] @@ -91,33 +80,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "161a24ea", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dest\n", - "ORD 17283\n", - "ATL 17215\n", - "LAX 16174\n", - "BOS 15508\n", - "MCO 14082\n", - " ... \n", - "MTJ 15\n", - "SBN 10\n", - "ANC 8\n", - "LEX 1\n", - "LGA 1\n", - "Name: count, Length: 105, dtype: int64" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights[\"dest\"].value_counts()" ] @@ -132,124 +98,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "8554277a", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
mean_delaycount_flights
dest
ORD13.57048417283
ATL12.50982417215
LAX9.40134416174
BOS8.73061315508
MCO11.27599814082
.........
MTJ17.64285715
SBN21.10000010
ANC12.8750008
LGANaN1
LEX-9.0000001
\n", - "

105 rows × 2 columns

\n", - "
" - ], - "text/plain": [ - " mean_delay count_flights\n", - "dest \n", - "ORD 13.570484 17283\n", - "ATL 12.509824 17215\n", - "LAX 9.401344 16174\n", - "BOS 8.730613 15508\n", - "MCO 11.275998 14082\n", - "... ... ...\n", - "MTJ 17.642857 15\n", - "SBN 21.100000 10\n", - "ANC 12.875000 8\n", - "LGA NaN 1\n", - "LEX -9.000000 1\n", - "\n", - "[105 rows x 2 columns]" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " flights.groupby([\"dest\"])\n", @@ -271,111 +123,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "061decae", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
miles
tailnum
D942DN3418
N0EGMQ250866
N10156115966
N102UW25722
N103US24619
......
N997DL54669
N998AT15432
N998DL66052
N999DN54623
N9EAMQ167317
\n", - "

4043 rows × 1 columns

\n", - "
" - ], - "text/plain": [ - " miles\n", - "tailnum \n", - "D942DN 3418\n", - "N0EGMQ 250866\n", - "N10156 115966\n", - "N102UW 25722\n", - "N103US 24619\n", - "... ...\n", - "N997DL 54669\n", - "N998AT 15432\n", - "N998DL 66052\n", - "N999DN 54623\n", - "N9EAMQ 167317\n", - "\n", - "[4043 rows x 1 columns]" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(flights.groupby(\"tailnum\").agg(miles=(\"distance\", \"sum\")))" ] @@ -390,111 +141,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "ecdb5630", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
n_cancelled
dest
ABQ0
ACK0
ALB20
ANC0
ATL317
......
TPA59
TUL16
TVC5
TYS52
XNA25
\n", - "

105 rows × 1 columns

\n", - "
" - ], - "text/plain": [ - " n_cancelled\n", - "dest \n", - "ABQ 0\n", - "ACK 0\n", - "ALB 20\n", - "ANC 0\n", - "ATL 317\n", - "... ...\n", - "TPA 59\n", - "TUL 16\n", - "TVC 5\n", - "TYS 52\n", - "XNA 25\n", - "\n", - "[105 rows x 1 columns]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(flights.groupby(\"dest\").agg(n_cancelled=(\"dep_time\", lambda x: x.isnull().sum())))" ] @@ -538,21 +188,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "a473cd56", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "np.int64(4983)" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights[\"distance\"].max()" ] @@ -567,67 +206,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "96285702", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
xy
013
152
27<NA>
\n", - "
" - ], - "text/plain": [ - " x y\n", - "0 1 3\n", - "1 5 2\n", - "2 7 " - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = pd.DataFrame({\"x\": [1, 5, 7], \"y\": [3, 2, pd.NA]})\n", "df" @@ -643,24 +225,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "5bae5499", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 1\n", - "1 2\n", - "2 7\n", - "dtype: object" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.min(axis=1)" ] @@ -677,23 +245,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "3af6ce86", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n", - "divided by 3 gives\n", - "remainder:\n", - "[1, 2, 0, 1, 2, 0, 1, 2, 0, 1]\n", - "divisions:\n", - "[0, 0, 1, 1, 1, 2, 2, 2, 3, 3]\n" - ] - } - ], + "outputs": [], "source": [ "print([x for x in range(1, 11)])\n", "print(\"divided by 3 gives\")\n", @@ -713,348 +268,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "6be57e34", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
yearmonthdaydep_timesched_dep_timedep_delayarr_timesched_arr_timearr_delaycarrierflighttailnumorigindestair_timedistancehourminutetime_hour
0201311517.05152.0830.081911.0UA1545N14228EWRIAH227.014005152013-01-01T10:00:00Z
1201311533.05294.0850.083020.0UA1714N24211LGAIAH227.014165292013-01-01T10:00:00Z
2201311542.05402.0923.085033.0AA1141N619AAJFKMIA160.010895402013-01-01T10:00:00Z
3201311544.0545-1.01004.01022-18.0B6725N804JBJFKBQN183.015765452013-01-01T10:00:00Z
4201311554.0600-6.0812.0837-25.0DL461N668DNLGAATL116.0762602013-01-01T11:00:00Z
............................................................
3367712013930NaN1455NaNNaN1634NaN9E3393NaNJFKDCANaN21314552013-09-30T18:00:00Z
3367722013930NaN2200NaNNaN2312NaN9E3525NaNLGASYRNaN1982202013-10-01T02:00:00Z
3367732013930NaN1210NaNNaN1330NaNMQ3461N535MQLGABNANaN76412102013-09-30T16:00:00Z
3367742013930NaN1159NaNNaN1344NaNMQ3572N511MQLGACLENaN41911592013-09-30T15:00:00Z
3367752013930NaN840NaNNaN1020NaNMQ3531N839MQLGARDUNaN4318402013-09-30T12:00:00Z
\n", - "

336776 rows × 19 columns

\n", - "
" - ], - "text/plain": [ - " year month day dep_time sched_dep_time dep_delay arr_time \\\n", - "0 2013 1 1 517.0 515 2.0 830.0 \n", - "1 2013 1 1 533.0 529 4.0 850.0 \n", - "2 2013 1 1 542.0 540 2.0 923.0 \n", - "3 2013 1 1 544.0 545 -1.0 1004.0 \n", - "4 2013 1 1 554.0 600 -6.0 812.0 \n", - "... ... ... ... ... ... ... ... \n", - "336771 2013 9 30 NaN 1455 NaN NaN \n", - "336772 2013 9 30 NaN 2200 NaN NaN \n", - "336773 2013 9 30 NaN 1210 NaN NaN \n", - "336774 2013 9 30 NaN 1159 NaN NaN \n", - "336775 2013 9 30 NaN 840 NaN NaN \n", - "\n", - " sched_arr_time arr_delay carrier flight tailnum origin dest \\\n", - "0 819 11.0 UA 1545 N14228 EWR IAH \n", - "1 830 20.0 UA 1714 N24211 LGA IAH \n", - "2 850 33.0 AA 1141 N619AA JFK MIA \n", - "3 1022 -18.0 B6 725 N804JB JFK BQN \n", - "4 837 -25.0 DL 461 N668DN LGA ATL \n", - "... ... ... ... ... ... ... ... \n", - "336771 1634 NaN 9E 3393 NaN JFK DCA \n", - "336772 2312 NaN 9E 3525 NaN LGA SYR \n", - "336773 1330 NaN MQ 3461 N535MQ LGA BNA \n", - "336774 1344 NaN MQ 3572 N511MQ LGA CLE \n", - "336775 1020 NaN MQ 3531 N839MQ LGA RDU \n", - "\n", - " air_time distance hour minute time_hour \n", - "0 227.0 1400 5 15 2013-01-01T10:00:00Z \n", - "1 227.0 1416 5 29 2013-01-01T10:00:00Z \n", - "2 160.0 1089 5 40 2013-01-01T10:00:00Z \n", - "3 183.0 1576 5 45 2013-01-01T10:00:00Z \n", - "4 116.0 762 6 0 2013-01-01T11:00:00Z \n", - "... ... ... ... ... ... \n", - "336771 NaN 213 14 55 2013-09-30T18:00:00Z \n", - "336772 NaN 198 22 0 2013-10-01T02:00:00Z \n", - "336773 NaN 764 12 10 2013-09-30T16:00:00Z \n", - "336774 NaN 419 11 59 2013-09-30T15:00:00Z \n", - "336775 NaN 431 8 40 2013-09-30T12:00:00Z \n", - "\n", - "[336776 rows x 19 columns]" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "flights.assign(\n", " hour=lambda x: x[\"sched_dep_time\"] // 100,\n", @@ -1074,79 +291,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "a3ffc085", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
yearmoney
02001105.000000
12002110.250000
22003115.762500
32004121.550625
42005127.628156
\n", - "
" - ], - "text/plain": [ - " year money\n", - "0 2001 105.000000\n", - "1 2002 110.250000\n", - "2 2003 115.762500\n", - "3 2004 121.550625\n", - "4 2005 127.628156" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import numpy as np\n", "\n", @@ -1168,821 +316,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "aaad2abb", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:35:12.580055\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "money.plot(x=\"year\", y=\"money\");" ] @@ -1997,767 +334,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "bbcf4d7c", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:35:12.673392\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "money.plot(x=\"year\", y=\"money\", logy=True);" ] @@ -2786,79 +366,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "ec1a09ca", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
yearmoney
02001105.00
12002110.25
22003115.76
32004121.55
42005127.63
\n", - "
" - ], - "text/plain": [ - " year money\n", - "0 2001 105.00\n", - "1 2002 110.25\n", - "2 2003 115.76\n", - "3 2004 121.55\n", - "4 2005 127.63" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "money.head().round(2)" ] @@ -2873,79 +384,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "9a306519", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
yearmoney
02001105.0
12002110.2
22003115.8
32004121.6
42005127.6
\n", - "
" - ], - "text/plain": [ - " year money\n", - "0 2001 105.0\n", - "1 2002 110.2\n", - "2 2003 115.8\n", - "3 2004 121.6\n", - "4 2005 127.6" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "money.head().round({\"year\": 0, \"money\": 1})" ] @@ -2960,79 +402,10 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "fb9dfb59", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
yearmoney
452046900.0
4620471000.0
4720481000.0
4820491100.0
4920501100.0
\n", - "
" - ], - "text/plain": [ - " year money\n", - "45 2046 900.0\n", - "46 2047 1000.0\n", - "47 2048 1000.0\n", - "48 2049 1100.0\n", - "49 2050 1100.0" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "money.tail().round({\"year\": 0, \"money\": -2})" ] @@ -3047,26 +420,10 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "f0290c0a", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 100.0\n", - "1 110.0\n", - "2 120.0\n", - "3 120.0\n", - "4 130.0\n", - "Name: money, dtype: float64" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "money[\"money\"].head().apply(lambda x: float(f'{float(f\"{x:.2g}\"):g}'))" ] @@ -3081,21 +438,10 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "6a6a4644", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([2., 2., 1.])" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "np.round([1.5, 2.5, 1.4])" ] @@ -3110,22 +456,10 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "6270c3cc", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([27.15101204, 15.41426421, 76.7650241 , 70.51407739, 88.22482077,\n", - " 51.56875497, 63.03191713, 86.16710762, 72.35185576, 87.16963342])" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "real_nums = 100 * np.random.random(size=10)\n", "real_nums" @@ -3133,42 +467,20 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "dc3608d6", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([28., 16., 77., 71., 89., 52., 64., 87., 73., 88.])" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "np.ceil(real_nums)" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "d577bb21", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([27., 15., 76., 70., 88., 51., 63., 86., 72., 87.])" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "np.floor(real_nums)" ] @@ -3183,26 +495,10 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "dca417b7", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 105.0\n", - "1 111.0\n", - "2 116.0\n", - "3 122.0\n", - "4 128.0\n", - "Name: money, dtype: float64" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "money[\"money\"].head().apply(np.ceil)" ] @@ -3219,26 +515,10 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "c5d77818", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "45 943.425818\n", - "46 1934.022928\n", - "47 2974.149892\n", - "48 4066.283205\n", - "49 5213.023184\n", - "Name: money, dtype: float64" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "money[\"money\"].tail().cumsum()" ] @@ -3267,134 +547,20 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "id": "db1272a3", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
xy
013
152
27<NA>
\n", - "
" - ], - "text/plain": [ - " x y\n", - "0 1 3\n", - "1 5 2\n", - "2 7 " - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "11ee0bac", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
xy
01.02.0
12.01.0
23.0NaN
\n", - "
" - ], - "text/plain": [ - " x y\n", - "0 1.0 2.0\n", - "1 2.0 1.0\n", - "2 3.0 NaN" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.rank()" ] @@ -3409,67 +575,10 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "edc7bd81", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
xy
00.3333331.0
10.6666670.5
21.000000NaN
\n", - "
" - ], - "text/plain": [ - " x y\n", - "0 0.333333 1.0\n", - "1 0.666667 0.5\n", - "2 1.000000 NaN" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.rank(pct=True)" ] @@ -3488,1078 +597,10 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "9e499d1b", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:35:12.825403\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "money[\"money_lag_5\"] = money[\"money\"].shift(5)\n", "money[\"money_lead_10\"] = money[\"money\"].shift(-10)\n", @@ -4602,21 +643,10 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "id": "81389031", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "np.float64(190.92197566022773)" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "money[\"money\"].quantile(0.25)" ] @@ -4631,25 +661,10 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "id": "9eb35866", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.00 105.000000\n", - "0.25 190.921976\n", - "0.50 347.101381\n", - "0.75 630.945970\n", - "Name: money, dtype: float64" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "money[\"money\"].quantile([0, 0.25, 0.5, 0.75])" ] @@ -4669,71 +684,10 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "id": "e2efca3a", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
distance_sdcount
origindest
EWREGE1.0110
JFKEGE1.0103
\n", - "
" - ], - "text/plain": [ - " distance_sd count\n", - "origin dest \n", - "EWR EGE 1.0 110\n", - "JFK EGE 1.0 103" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " flights.groupby([\"origin\", \"dest\"])\n", @@ -4759,3173 +713,20 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "id": "b5ac4861", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:35:13.026671\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "flights[\"dep_delay\"].plot.hist(bins=50, title=\" Distribution: length of delay\");" ] }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "id": "d4e76051", "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " 2026-04-28T13:35:13.184191\n", - " image/svg+xml\n", - " \n", - " \n", - " Matplotlib v3.10.0, https://matplotlib.org/\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n" - ], - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "flights.query(\"dep_delay <= 120\")[\"dep_delay\"].plot.hist(\n", " bins=50, title=\" Distribution: length of delay\"\n", diff --git a/rectangling.ipynb b/rectangling.ipynb index 4c0ff4f..2ddf375 100644 --- a/rectangling.ipynb +++ b/rectangling.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "51a55374", "metadata": { "tags": [ @@ -56,18 +56,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "4d1c3fa4", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[10, 1.23, 'like this', True, None]\n" - ] - } - ], + "outputs": [], "source": [ "list_example = [10, 1.23, \"like this\", True, None]\n", "print(list_example)" @@ -83,18 +75,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "96c4b86b", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[10, 1.23, 'like this', True, None, 'one more entry']\n" - ] - } - ], + "outputs": [], "source": [ "list_example.append(\"one more entry\")\n", "print(list_example)" @@ -110,19 +94,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "138ac0e0", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "10\n", - "one more entry\n" - ] - } - ], + "outputs": [], "source": [ "print(list_example[0])\n", "print(list_example[-1])" @@ -148,19 +123,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "ef390263", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[10, 1.23, 'like this']\n", - "[True, None, 'one more entry']\n" - ] - } - ], + "outputs": [], "source": [ "print(list_example[:3])\n", "print(list_example[-3:])" @@ -176,18 +142,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "13584953", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2, 4, 6, 8]\n" - ] - } - ], + "outputs": [], "source": [ "list_of_numbers = list(range(1, 11))\n", "start = 1\n", @@ -206,18 +164,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "f2bc8926", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[10, 9, 8, 7, 6, 5, 4, 3, 2, 1]\n" - ] - } - ], + "outputs": [], "source": [ "print(list_of_numbers[::-1])" ] @@ -242,28 +192,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "703fb0b2", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[3.1415,\n", - " 16,\n", - " ['five', 4, 3],\n", - " (91, 93, 90),\n", - " 'Hello World!',\n", - " True,\n", - " None,\n", - " {'key': 'value', 'key2': 'value2'}]" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "wacky_list = [\n", " 3.1415,\n", @@ -290,21 +222,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "28fe13d2", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15]]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "multilayer_list = [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15]]\n", "multilayer_list" @@ -320,21 +241,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "73a71803", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "[x for little_list in multilayer_list for x in little_list]" ] @@ -359,7 +269,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "007dadfb", "metadata": {}, "outputs": [], @@ -377,7 +287,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "aeb70f97", "metadata": {}, "outputs": [], @@ -397,41 +307,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "fa1d828c", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (4, 3)
alphabetagamma
stri64list[str]
"["0,1,2"]"1["a", "b", "c"]
"foo"1null
"[]"1[]
"["3,4"]"1["d", "e"]
" - ], - "text/plain": [ - "shape: (4, 3)\n", - "┌───────────┬──────┬─────────────────┐\n", - "│ alpha ┆ beta ┆ gamma │\n", - "│ --- ┆ --- ┆ --- │\n", - "│ str ┆ i64 ┆ list[str] │\n", - "╞═══════════╪══════╪═════════════════╡\n", - "│ [\"0,1,2\"] ┆ 1 ┆ [\"a\", \"b\", \"c\"] │\n", - "│ foo ┆ 1 ┆ null │\n", - "│ [] ┆ 1 ┆ [] │\n", - "│ [\"3,4\"] ┆ 1 ┆ [\"d\", \"e\"] │\n", - "└───────────┴──────┴─────────────────┘" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = pl.DataFrame(\n", " {\n", @@ -454,44 +333,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "0854bcf3", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (7, 3)
alphabetagamma
stri64str
"["0,1,2"]"1"a"
"["0,1,2"]"1"b"
"["0,1,2"]"1"c"
"foo"1null
"[]"1null
"["3,4"]"1"d"
"["3,4"]"1"e"
" - ], - "text/plain": [ - "shape: (7, 3)\n", - "┌───────────┬──────┬───────┐\n", - "│ alpha ┆ beta ┆ gamma │\n", - "│ --- ┆ --- ┆ --- │\n", - "│ str ┆ i64 ┆ str │\n", - "╞═══════════╪══════╪═══════╡\n", - "│ [\"0,1,2\"] ┆ 1 ┆ a │\n", - "│ [\"0,1,2\"] ┆ 1 ┆ b │\n", - "│ [\"0,1,2\"] ┆ 1 ┆ c │\n", - "│ foo ┆ 1 ┆ null │\n", - "│ [] ┆ 1 ┆ null │\n", - "│ [\"3,4\"] ┆ 1 ┆ d │\n", - "│ [\"3,4\"] ┆ 1 ┆ e │\n", - "└───────────┴──────┴───────┘" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df.explode(\"gamma\")" ] @@ -539,7 +384,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "285ade5e", "metadata": {}, "outputs": [], @@ -562,21 +407,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "75399bf0", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "type(json_data)" ] @@ -593,36 +427,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "8cc4f408", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[{'date': '1971 FEB',\n", - " 'value': '3.8',\n", - " 'label': '1971 JAN-MAR',\n", - " 'year': '1971',\n", - " 'month': 'February',\n", - " 'quarter': '',\n", - " 'sourceDataset': 'LMS',\n", - " 'updateDate': '2015-10-13T23:00:00.000Z'},\n", - " {'date': '1971 MAR',\n", - " 'value': '3.9',\n", - " 'label': '1971 FEB-APR',\n", - " 'year': '1971',\n", - " 'month': 'March',\n", - " 'quarter': '',\n", - " 'sourceDataset': 'LMS',\n", - " 'updateDate': '2015-10-13T23:00:00.000Z'}]" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "json_data[\"months\"][:2]" ] @@ -639,7 +447,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "05069731", "metadata": {}, "outputs": [], @@ -665,24 +473,10 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "eca7982f", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'food': 'doughnut',\n", - " 'good_with': ['coffee', 'tea'],\n", - " 'flavour': None,\n", - " 'toppings': [{'id': 0, 'type': 'glazed'}, {'id': 1, 'type': 'sugar'}]}" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import json\n", "\n", @@ -700,21 +494,10 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "ea6f887d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'{\"food\": \"doughnut\", \"good_with\": [\"coffee\", \"tea\"], \"flavour\": null, \"toppings\": [{\"id\": 0, \"type\": \"glazed\"}, {\"id\": 1, \"type\": \"sugar\"}]}'" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "json_stream = json.dumps(result)\n", "json_stream" @@ -763,7 +546,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "1dbfd7c3", "metadata": {}, "outputs": [], @@ -783,40 +566,10 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "b3ea0e0b", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (3, 6)
idname.firstname.lastname.givenname.familyname
i64strstrstrstrstr
1"Coleen""Volk"nullnullnull
nullnullnull"Mark""Regner"null
2nullnullnullnull"Faye Raker"
" - ], - "text/plain": [ - "shape: (3, 6)\n", - "┌──────┬────────────┬───────────┬────────────┬─────────────┬────────────┐\n", - "│ id ┆ name.first ┆ name.last ┆ name.given ┆ name.family ┆ name │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ str ┆ str ┆ str ┆ str ┆ str │\n", - "╞══════╪════════════╪═══════════╪════════════╪═════════════╪════════════╡\n", - "│ 1 ┆ Coleen ┆ Volk ┆ null ┆ null ┆ null │\n", - "│ null ┆ null ┆ null ┆ Mark ┆ Regner ┆ null │\n", - "│ 2 ┆ null ┆ null ┆ null ┆ null ┆ Faye Raker │\n", - "└──────┴────────────┴───────────┴────────────┴─────────────┴────────────┘" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "data = [\n", " {\"id\": 1, \"name\": {\"first\": \"Coleen\", \"last\": \"Volk\"}},\n", @@ -836,40 +589,10 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "9349a3a7", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (3, 2)
idname
i64str
1"{"first": "Coleen", "last": "V…
null"{"given": "Mark", "family": "R…
2"Faye Raker"
" - ], - "text/plain": [ - "shape: (3, 2)\n", - "┌──────┬─────────────────────────────────┐\n", - "│ id ┆ name │\n", - "│ --- ┆ --- │\n", - "│ i64 ┆ str │\n", - "╞══════╪═════════════════════════════════╡\n", - "│ 1 ┆ {\"first\": \"Coleen\", \"last\": \"V… │\n", - "│ null ┆ {\"given\": \"Mark\", \"family\": \"R… │\n", - "│ 2 ┆ Faye Raker │\n", - "└──────┴─────────────────────────────────┘" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pl.json_normalize(data, max_level=0)" ] diff --git a/regex.ipynb b/regex.ipynb index d8cb9d0..520a8c8 100644 --- a/regex.ipynb +++ b/regex.ipynb @@ -23,30 +23,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "<>:4: SyntaxWarning: invalid escape sequence '\\w'\n", - "<>:4: SyntaxWarning: invalid escape sequence '\\w'\n", - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60848/970224743.py:4: SyntaxWarning: invalid escape sequence '\\w'\n", - " re.findall(\"string \\w+\\s\", text)\n" - ] - }, - { - "data": { - "text/plain": [ - "['string cleaning ', 'string editing ']" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import re\n", "\n", @@ -78,30 +57,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "<>:2: SyntaxWarning: invalid escape sequence '\\w'\n", - "<>:2: SyntaxWarning: invalid escape sequence '\\w'\n", - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60848/3231031485.py:2: SyntaxWarning: invalid escape sequence '\\w'\n", - " re.sub(\"string \\w+\\s\", new_text, text)\n" - ] - }, - { - "data": { - "text/plain": [ - "'It is true that new text here! is a topic in this chapter. new text here! is another.'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "new_text = \"new text here! \"\n", "re.sub(\"string \\w+\\s\", new_text, text)" @@ -183,18 +141,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The greedy match is stackoverflo\n", - "The lazy match is stacko\n" - ] - } - ], + "outputs": [], "source": [ "test_string = \"stackoverflow\"\n", "greedy_regex = \"s.*o\"\n", @@ -222,30 +171,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "<>:2: SyntaxWarning: invalid escape sequence '\\$'\n", - "<>:2: SyntaxWarning: invalid escape sequence '\\$'\n", - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60848/3729052804.py:2: SyntaxWarning: invalid escape sequence '\\$'\n", - " re.findall(\"\\$(\\d{2}.\\d{2})\", text)\n" - ] - }, - { - "data": { - "text/plain": [ - "['45.34', '50.00']" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "text = \"Product 1 was $45.34, while product 2 came in at $50.00 however it was assessed that the $4.66 difference did not make up for the higher quality of product 2.\"\n", "re.findall(\"\\$(\\d{2}.\\d{2})\", text)" @@ -264,20 +192,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['30500.00 to 35000']" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "sal_r_per = r\"\\b([0-9]{1,6}(?:\\.)?(?:[0-9]{1,2})?(?:\\s?-\\s?|\\s?to\\s?)[0-9]{1,6}(?:\\.)?(?:[0-9]{1,2})?)(?:\\s?per)\\b\"\n", "text = \"This job pays gbp 30500.00 to 35000 per year. Apply at number 100 per the below address.\"\n", diff --git a/spreadsheets.ipynb b/spreadsheets.ipynb index 4f5ebfd..6d62900 100644 --- a/spreadsheets.ipynb +++ b/spreadsheets.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "51a55374", "metadata": { "tags": [ @@ -68,43 +68,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "bf00d4c6", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 5)
Student IDFull Namefavourite.foodmealPlanAGE
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only"null
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""five"
6"Güvenç Attila""Ice cream""Lunch only""6"
" - ], - "text/plain": [ - "shape: (6, 5)\n", - "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", - "│ Student ID ┆ Full Name ┆ favourite.food ┆ mealPlan ┆ AGE │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", - "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", - "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", - "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", - "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", - "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ null │\n", - "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ five │\n", - "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", - "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import polars as pl\n", "\n", @@ -126,43 +93,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "ecce26ea", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 5)
student_idfull_namefavourite_foodmeal_planage
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only"null
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""five"
6"Güvenç Attila""Ice cream""Lunch only""6"
" - ], - "text/plain": [ - "shape: (6, 5)\n", - "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", - "│ student_id ┆ full_name ┆ favourite_food ┆ meal_plan ┆ age │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", - "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", - "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", - "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", - "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", - "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ null │\n", - "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ five │\n", - "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", - "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "students.columns = [\n", " \"student_id\",\n", @@ -184,43 +118,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "22e45cc2", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (6, 5)
student_idfull_namefavourite_foodmeal_planage
i64strstrstrstr
1"Sunil Huffmann""Strawberry yoghurt""Lunch only""4"
2"Barclay Lynn""French fries""Lunch only""5"
3"Jayendra Lyne""N/A""Breakfast and lunch""7"
4"Leon Rossini""Anchovies""Lunch only"null
5"Chidiegwu Dunkel""Pizza""Breakfast and lunch""5"
6"Güvenç Attila""Ice cream""Lunch only""6"
" - ], - "text/plain": [ - "shape: (6, 5)\n", - "┌────────────┬──────────────────┬────────────────────┬─────────────────────┬──────┐\n", - "│ student_id ┆ full_name ┆ favourite_food ┆ meal_plan ┆ age │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ str ┆ str ┆ str ┆ str │\n", - "╞════════════╪══════════════════╪════════════════════╪═════════════════════╪══════╡\n", - "│ 1 ┆ Sunil Huffmann ┆ Strawberry yoghurt ┆ Lunch only ┆ 4 │\n", - "│ 2 ┆ Barclay Lynn ┆ French fries ┆ Lunch only ┆ 5 │\n", - "│ 3 ┆ Jayendra Lyne ┆ N/A ┆ Breakfast and lunch ┆ 7 │\n", - "│ 4 ┆ Leon Rossini ┆ Anchovies ┆ Lunch only ┆ null │\n", - "│ 5 ┆ Chidiegwu Dunkel ┆ Pizza ┆ Breakfast and lunch ┆ 5 │\n", - "│ 6 ┆ Güvenç Attila ┆ Ice cream ┆ Lunch only ┆ 6 │\n", - "└────────────┴──────────────────┴────────────────────┴─────────────────────┴──────┘" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "students = pl.read_excel(\"data/students.xlsx\")\n", "students.columns = [\n", @@ -244,25 +145,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "f67490d3", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Schema([('student_id', Int64),\n", - " ('full_name', String),\n", - " ('favourite_food', String),\n", - " ('meal_plan', Categorical(ordering='physical')),\n", - " ('age', Int64)])" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "students = students.with_columns(\n", " [\n", @@ -300,43 +186,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "e37f9e3d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (5, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64i64i64stri64
"Adelie""Torgersen"39.118.71813750"male"2007
"Adelie""Torgersen"39.517.41863800"female"2007
"Adelie""Torgersen"40.318.01953250"female"2007
"Adelie""Torgersen"nullnullnullnull"NA"2007
"Adelie""Torgersen"36.719.31933450"female"2007
" - ], - "text/plain": [ - "shape: (5, 8)\n", - "┌─────────┬───────────┬───────────────┬───────────────┬──────────────┬─────────────┬────────┬──────┐\n", - "│ species ┆ island ┆ bill_length_m ┆ bill_depth_mm ┆ flipper_leng ┆ body_mass_g ┆ sex ┆ year │\n", - "│ --- ┆ --- ┆ m ┆ --- ┆ th_mm ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ --- ┆ f64 ┆ --- ┆ i64 ┆ str ┆ i64 │\n", - "│ ┆ ┆ f64 ┆ ┆ i64 ┆ ┆ ┆ │\n", - "╞═════════╪═══════════╪═══════════════╪═══════════════╪══════════════╪═════════════╪════════╪══════╡\n", - "│ Adelie ┆ Torgersen ┆ 39.1 ┆ 18.7 ┆ 181 ┆ 3750 ┆ male ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 39.5 ┆ 17.4 ┆ 186 ┆ 3800 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 40.3 ┆ 18.0 ┆ 195 ┆ 3250 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ null ┆ null ┆ null ┆ null ┆ NA ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 36.7 ┆ 19.3 ┆ 193 ┆ 3450 ┆ female ┆ 2007 │\n", - "└─────────┴───────────┴───────────────┴───────────────┴──────────────┴─────────────┴────────┴──────┘" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pl.read_excel(\n", " \"data/penguins.xlsx\",\n", @@ -356,18 +209,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "0f3a3b78", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['Torgersen Island', 'Biscoe Island', 'Dream Island']\n" - ] - } - ], + "outputs": [], "source": [ "penguins_dict = pl.read_excel(\n", " \"data/penguins.xlsx\",\n", @@ -386,50 +231,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "15495426", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Biscoe Island\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "shape: (5, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64i64i64stri64
"Adelie""Biscoe"37.818.31743400"female"2007
"Adelie""Biscoe"37.718.71803600"male"2007
"Adelie""Biscoe"35.919.21893800"female"2007
"Adelie""Biscoe"38.218.11853950"male"2007
"Adelie""Biscoe"38.817.21803800"male"2007
" - ], - "text/plain": [ - "shape: (5, 8)\n", - "┌─────────┬────────┬────────────────┬───────────────┬────────────────┬─────────────┬────────┬──────┐\n", - "│ species ┆ island ┆ bill_length_mm ┆ bill_depth_mm ┆ flipper_length ┆ body_mass_g ┆ sex ┆ year │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ _mm ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ f64 ┆ f64 ┆ --- ┆ i64 ┆ str ┆ i64 │\n", - "│ ┆ ┆ ┆ ┆ i64 ┆ ┆ ┆ │\n", - "╞═════════╪════════╪════════════════╪═══════════════╪════════════════╪═════════════╪════════╪══════╡\n", - "│ Adelie ┆ Biscoe ┆ 37.8 ┆ 18.3 ┆ 174 ┆ 3400 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Biscoe ┆ 37.7 ┆ 18.7 ┆ 180 ┆ 3600 ┆ male ┆ 2007 │\n", - "│ Adelie ┆ Biscoe ┆ 35.9 ┆ 19.2 ┆ 189 ┆ 3800 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Biscoe ┆ 38.2 ┆ 18.1 ┆ 185 ┆ 3950 ┆ male ┆ 2007 │\n", - "│ Adelie ┆ Biscoe ┆ 38.8 ┆ 17.2 ┆ 180 ┆ 3800 ┆ male ┆ 2007 │\n", - "└─────────┴────────┴────────────────┴───────────────┴────────────────┴─────────────┴────────┴──────┘" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "print(list(penguins_dict.keys())[1])\n", "list(penguins_dict.values())[1].head()" @@ -445,49 +250,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "151ba846", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (344, 8)
speciesislandbill_length_mmbill_depth_mmflipper_length_mmbody_mass_gsexyear
strstrf64f64i64i64stri64
"Adelie""Torgersen"39.118.71813750"male"2007
"Adelie""Torgersen"39.517.41863800"female"2007
"Adelie""Torgersen"40.318.01953250"female"2007
"Adelie""Torgersen"nullnullnullnull"NA"2007
"Adelie""Torgersen"36.719.31933450"female"2007
……………………
"Chinstrap""Dream"55.819.82074000"male"2009
"Chinstrap""Dream"43.518.12023400"female"2009
"Chinstrap""Dream"49.618.21933775"male"2009
"Chinstrap""Dream"50.819.02104100"male"2009
"Chinstrap""Dream"50.218.71983775"female"2009
" - ], - "text/plain": [ - "shape: (344, 8)\n", - "┌───────────┬───────────┬──────────────┬──────────────┬──────────────┬─────────────┬────────┬──────┐\n", - "│ species ┆ island ┆ bill_length_ ┆ bill_depth_m ┆ flipper_leng ┆ body_mass_g ┆ sex ┆ year │\n", - "│ --- ┆ --- ┆ mm ┆ m ┆ th_mm ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ --- ┆ --- ┆ --- ┆ i64 ┆ str ┆ i64 │\n", - "│ ┆ ┆ f64 ┆ f64 ┆ i64 ┆ ┆ ┆ │\n", - "╞═══════════╪═══════════╪══════════════╪══════════════╪══════════════╪═════════════╪════════╪══════╡\n", - "│ Adelie ┆ Torgersen ┆ 39.1 ┆ 18.7 ┆ 181 ┆ 3750 ┆ male ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 39.5 ┆ 17.4 ┆ 186 ┆ 3800 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 40.3 ┆ 18.0 ┆ 195 ┆ 3250 ┆ female ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ null ┆ null ┆ null ┆ null ┆ NA ┆ 2007 │\n", - "│ Adelie ┆ Torgersen ┆ 36.7 ┆ 19.3 ┆ 193 ┆ 3450 ┆ female ┆ 2007 │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ Chinstrap ┆ Dream ┆ 55.8 ┆ 19.8 ┆ 207 ┆ 4000 ┆ male ┆ 2009 │\n", - "│ Chinstrap ┆ Dream ┆ 43.5 ┆ 18.1 ┆ 202 ┆ 3400 ┆ female ┆ 2009 │\n", - "│ Chinstrap ┆ Dream ┆ 49.6 ┆ 18.2 ┆ 193 ┆ 3775 ┆ male ┆ 2009 │\n", - "│ Chinstrap ┆ Dream ┆ 50.8 ┆ 19.0 ┆ 210 ┆ 4100 ┆ male ┆ 2009 │\n", - "│ Chinstrap ┆ Dream ┆ 50.2 ┆ 18.7 ┆ 198 ┆ 3775 ┆ female ┆ 2009 │\n", - "└───────────┴───────────┴──────────────┴──────────────┴──────────────┴─────────────┴────────┴──────┘" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "penguins = pl.concat(penguins_dict.values())\n", "penguins" @@ -519,49 +285,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "29987b9d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (14, 6)
Lots of people__UNNAMED__1__UNNAMED__2__UNNAMED__3__UNNAMED__4__UNNAMED__5
strstrstrstrdatestr
"David Bowie""musician""69""true"1947-01-08"2016-01-10 00:00:00"
"Carrie Fisher""actor""60""true"1956-10-21"2016-12-27 00:00:00"
"Chuck Berry""musician""90""true"1926-10-18"2017-03-18 00:00:00"
"Bill Paxton""actor""61""true"1955-05-17"2017-02-25 00:00:00"
"Prince""musician""57""true"1958-06-07"2016-04-21 00:00:00"
………………
"George Michael""musician""53""false"1963-06-25"2016-12-25 00:00:00"
"Some"nullnullnullnullnull
null"also like to write stuff"nullnullnullnull
nullnull"at the""bottom,"nullnull
nullnullnullnullnull"too!"
" - ], - "text/plain": [ - "shape: (14, 6)\n", - "┌────────────────┬──────────────┬──────────────┬──────────────┬──────────────┬─────────────────────┐\n", - "│ Lots of people ┆ __UNNAMED__1 ┆ __UNNAMED__2 ┆ __UNNAMED__3 ┆ __UNNAMED__4 ┆ __UNNAMED__5 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ str ┆ str ┆ date ┆ str │\n", - "╞════════════════╪══════════════╪══════════════╪══════════════╪══════════════╪═════════════════════╡\n", - "│ David Bowie ┆ musician ┆ 69 ┆ true ┆ 1947-01-08 ┆ 2016-01-10 00:00:00 │\n", - "│ Carrie Fisher ┆ actor ┆ 60 ┆ true ┆ 1956-10-21 ┆ 2016-12-27 00:00:00 │\n", - "│ Chuck Berry ┆ musician ┆ 90 ┆ true ┆ 1926-10-18 ┆ 2017-03-18 00:00:00 │\n", - "│ Bill Paxton ┆ actor ┆ 61 ┆ true ┆ 1955-05-17 ┆ 2017-02-25 00:00:00 │\n", - "│ Prince ┆ musician ┆ 57 ┆ true ┆ 1958-06-07 ┆ 2016-04-21 00:00:00 │\n", - "│ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", - "│ George Michael ┆ musician ┆ 53 ┆ false ┆ 1963-06-25 ┆ 2016-12-25 00:00:00 │\n", - "│ Some ┆ null ┆ null ┆ null ┆ null ┆ null │\n", - "│ null ┆ also like to ┆ null ┆ null ┆ null ┆ null │\n", - "│ ┆ write stuff ┆ ┆ ┆ ┆ │\n", - "│ null ┆ null ┆ at the ┆ bottom, ┆ null ┆ null │\n", - "│ null ┆ null ┆ null ┆ null ┆ null ┆ too! │\n", - "└────────────────┴──────────────┴──────────────┴──────────────┴──────────────┴─────────────────────┘" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pl.read_excel(\n", " \"data/deaths.xlsx\",\n", @@ -579,47 +306,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "9d7a3db3", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (10, 6)
Lots of people__UNNAMED__1__UNNAMED__2__UNNAMED__3__UNNAMED__4__UNNAMED__5
strstri64booldatedate
"David Bowie""musician"69true1947-01-082016-01-10
"Carrie Fisher""actor"60true1956-10-212016-12-27
"Chuck Berry""musician"90true1926-10-182017-03-18
"Bill Paxton""actor"61true1955-05-172017-02-25
"Prince""musician"57true1958-06-072016-04-21
"Alan Rickman""actor"69false1946-02-212016-01-14
"Florence Henderson""actor"82true1934-02-142016-11-24
"Harper Lee""author"89false1926-04-282016-02-19
"Zsa Zsa Gábor""actor"99true1917-02-062016-12-18
"George Michael""musician"53false1963-06-252016-12-25
" - ], - "text/plain": [ - "shape: (10, 6)\n", - "┌────────────────────┬──────────────┬──────────────┬──────────────┬──────────────┬──────────────┐\n", - "│ Lots of people ┆ __UNNAMED__1 ┆ __UNNAMED__2 ┆ __UNNAMED__3 ┆ __UNNAMED__4 ┆ __UNNAMED__5 │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ i64 ┆ bool ┆ date ┆ date │\n", - "╞════════════════════╪══════════════╪══════════════╪══════════════╪══════════════╪══════════════╡\n", - "│ David Bowie ┆ musician ┆ 69 ┆ true ┆ 1947-01-08 ┆ 2016-01-10 │\n", - "│ Carrie Fisher ┆ actor ┆ 60 ┆ true ┆ 1956-10-21 ┆ 2016-12-27 │\n", - "│ Chuck Berry ┆ musician ┆ 90 ┆ true ┆ 1926-10-18 ┆ 2017-03-18 │\n", - "│ Bill Paxton ┆ actor ┆ 61 ┆ true ┆ 1955-05-17 ┆ 2017-02-25 │\n", - "│ Prince ┆ musician ┆ 57 ┆ true ┆ 1958-06-07 ┆ 2016-04-21 │\n", - "│ Alan Rickman ┆ actor ┆ 69 ┆ false ┆ 1946-02-21 ┆ 2016-01-14 │\n", - "│ Florence Henderson ┆ actor ┆ 82 ┆ true ┆ 1934-02-14 ┆ 2016-11-24 │\n", - "│ Harper Lee ┆ author ┆ 89 ┆ false ┆ 1926-04-28 ┆ 2016-02-19 │\n", - "│ Zsa Zsa Gábor ┆ actor ┆ 99 ┆ true ┆ 1917-02-06 ┆ 2016-12-18 │\n", - "│ George Michael ┆ musician ┆ 53 ┆ false ┆ 1963-06-25 ┆ 2016-12-25 │\n", - "└────────────────────┴──────────────┴──────────────┴──────────────┴──────────────┴──────────────┘" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pl.read_excel(\n", " \"data/deaths.xlsx\",\n", @@ -666,40 +356,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "15963e18", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (3, 2)
itemquantity
cati64
"brownie"10
"cupcake"5
"cookie"8
" - ], - "text/plain": [ - "shape: (3, 2)\n", - "┌─────────┬──────────┐\n", - "│ item ┆ quantity │\n", - "│ --- ┆ --- │\n", - "│ cat ┆ i64 │\n", - "╞═════════╪══════════╡\n", - "│ brownie ┆ 10 │\n", - "│ cupcake ┆ 5 │\n", - "│ cookie ┆ 8 │\n", - "└─────────┴──────────┘" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "bake_sale = pl.DataFrame(\n", " {\n", @@ -720,21 +380,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "1fc17141", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "bake_sale.write_excel(\"data/bake_sale.xlsx\")" ] @@ -759,21 +408,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "27e128f9", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Schema([('item', String), ('quantity', Int64)])" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pl.read_excel(\"data/bake_sale.xlsx\").schema" ] diff --git a/strings.ipynb b/strings.ipynb index 10d93ae..b1a00fe 100644 --- a/strings.ipynb +++ b/strings.ipynb @@ -28,7 +28,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "478a847a", "metadata": {}, "outputs": [], @@ -49,21 +49,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "d7f4ea2d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "str" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "type(string_one)" ] @@ -78,7 +67,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "01379fe7", "metadata": {}, "outputs": [], @@ -88,21 +77,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "d88f7928", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'ban'" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "var[:3]" ] @@ -117,21 +95,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "e03d95d1", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'aaa'" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "var[1::2]" ] @@ -148,21 +115,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "83ab201b", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "6" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "len(var)" ] @@ -177,21 +133,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "7801bd5d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\"This is a string. If I want to include a 'quote' inside a string, I use double quotes on the outside.\"" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "string_one + \". \" + string_two + \".\"" ] @@ -206,21 +151,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "138cef18", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\"This is a string. If I want to include a 'quote' inside a string, I use double quotes on the outside\"" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "\". \".join([string_one, string_two])" ] @@ -235,18 +169,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "e11896f8", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['INPUT TEXT', 'input text', 'Input Text']\n" - ] - } - ], + "outputs": [], "source": [ "var = \"input TEXT\"\n", "var_list = [var.upper(), var.lower(), var.title()]\n", @@ -275,20 +201,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "bf0aadec", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INPUT TEXT; and \n", - "input text; and \n", - "Input Text\n" - ] - } - ], + "outputs": [], "source": [ "print(*var_list, sep=\"; and \\n\")" ] @@ -303,21 +219,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "a96f048c", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'A boolean is either True or False, there are only 2 options.'" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " \"A boolean is either \"\n", @@ -350,18 +255,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "9dddf0da", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "You scored 15.32399\n" - ] - } - ], + "outputs": [], "source": [ "variable = 15.32399\n", "print(f\"You scored {variable}\")" @@ -377,18 +274,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "795e7c07", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "You scored 234.8246695201\n" - ] - } - ], + "outputs": [], "source": [ "print(f\"You scored {variable**2}\")" ] @@ -411,18 +300,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "1f3d3806", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "You scored +15.32\n" - ] - } - ], + "outputs": [], "source": [ "print(f\"You scored {variable:+.2f}\")" ] @@ -471,21 +352,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "0ccd65aa", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import string\n", "\n", @@ -502,21 +372,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "16205c36", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "string.ascii_letters" ] @@ -531,21 +390,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "0c67f5cd", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'0123456789'" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "string.digits" ] @@ -568,20 +416,10 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "16e9904a", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Here is a \n", - " new line\n", - "Here is an \\n escaped new line \n" - ] - } - ], + "outputs": [], "source": [ "print(\"Here is a \\n new line\")\n", "print(\"Here is an \\\\n escaped new line \")" @@ -614,19 +452,10 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "af423bd1", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "a\tb\n", - "A\tB\n" - ] - } - ], + "outputs": [], "source": [ "print(\"a\\tb\\nA\\tB\")" ] @@ -643,18 +472,10 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "c2b9c689", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "a\\tb\\nA\\tB\n" - ] - } - ], + "outputs": [], "source": [ "print(r\"a\\tb\\nA\\tB\")" ] @@ -675,21 +496,10 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "229ada3a", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Value is subjective'" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "\"Value is objective\".replace(\"objective\", \"subjective\")" ] @@ -704,21 +514,10 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "79f754dc", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Value is subjective'" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "text = \"Value is objective\"\n", "old_substr = \"objective\"\n", @@ -750,21 +549,10 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "99675fee", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'a': '', 'e': '', 'i': '', 'o': '', 'u': ''}" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "example_text = \"Much recent work has focused on the influence of social capital on innovative outcomes. Little research has been done on disadvantaged groups who were often restricted from participation in social networks that provide information necessary for invention and innovation. Unique new data on African American inventors and patentees between 1843 and 1930 permit an empirical investigation of the relation between social capital and economic outcomes. I find that African Americans used both traditional, i.e., occupation-based, and nontraditional, i.e., civic, networks to maximize inventive output and that laws constraining social-capital formation are most negatively correlated with economically important inventive activity.\"\n", "vowels = \"aeiou\"\n", @@ -782,21 +570,10 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "e48763cb", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Mch rcnt wrk hs fcsd n th nflnc f scl cptl n nnvtv tcms. Lttl rsrch hs bn dn n dsdvntgd grps wh wr ftn rstrctd frm prtcptn n scl ntwrks tht prvd nfrmtn ncssry fr nvntn nd nnvtn. Unq nw dt n Afrcn Amrcn nvntrs nd ptnts btwn 1843 nd 1930 prmt n mprcl nvstgtn f th rltn btwn scl cptl nd cnmc tcms. I fnd tht Afrcn Amrcns sd bth trdtnl, .., ccptn-bsd, nd nntrdtnl, .., cvc, ntwrks t mxmz nvntv tpt nd tht lws cnstrnng scl-cptl frmtn r mst ngtvly crrltd wth cnmclly mprtnt nvntv ctvty.'" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "translator = example_text.maketrans(translation_dict)\n", "example_text.translate(translator)" @@ -822,21 +599,10 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "ac758b38", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Example string with excess punctuation'" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "\"\".join(\n", " [\n", @@ -859,21 +625,10 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "id": "122619bf", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'This is a sentence'" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "\"This is a sentence and we will split it at character 18\"[:18]" ] @@ -888,21 +643,10 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "9fc432ed", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['This is a sentence', ' And another sentence', ' And a third sentence']" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "\"This is a sentence. And another sentence. And a third sentence\".split(\".\")" ] @@ -917,21 +661,10 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "6904e486", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['This is a ', '. And another ', '. And a third ', '']" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "\"This is a sentence. And another sentence. And a third sentence\".split(\"sentence\")" ] @@ -956,18 +689,10 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "22f94993", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The word \"coffee\" appears 2 times.\n" - ] - } - ], + "outputs": [], "source": [ "text = \"At six o'clock we were waiting for coffee, \\n waiting for coffee and the charitable crumb \\n that was going to be served from a certain balcony \\n --like kings of old, or like a miracle. \\n It was still dark. One foot of the sun \\n steadied itself on a long ripple in the river.\"\n", "word = \"coffee\"\n", @@ -984,21 +709,10 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "id": "a351a11b", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "35" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "text.find(word)" ] @@ -1013,21 +727,10 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "id": "8e0a7020", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'coffee'" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "text[text.find(word) : text.find(word) + len(word)]" ] @@ -1042,21 +745,10 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "id": "e18f64a3", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "57" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "text.rfind(word)" ] @@ -1083,21 +775,10 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "id": "bbc3eb7b", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['Ada', 'Adam', 'Elinor', 'Grace', 'Jean']" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "[name.capitalize() for name in [\"ada\", \"adam\", \"elinor\", \"grace\", \"jean\"]]" ] @@ -1112,26 +793,10 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "id": "c8a7f68b", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 ada lovelace\n", - "1 adam smith\n", - "2 elinor ostrom\n", - "3 grace hopper\n", - "4 jean bartik\n", - "dtype: string" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import pandas as pd\n", "\n", @@ -1152,26 +817,10 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "id": "7cf149b5", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 Ada Lovelace\n", - "1 Adam Smith\n", - "2 Elinor Ostrom\n", - "3 Grace Hopper\n", - "4 Jean Bartik\n", - "dtype: string" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dfs.str.title()" ] @@ -1186,26 +835,10 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "id": "26dc9a7b", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 Ada Lovelace\n", - "1 Adam Smith\n", - "2 Elinor Ostrom\n", - "3 Grace Hopper\n", - "4 Jean Bartik\n", - "Name: names, dtype: string" - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = pd.DataFrame(dfs, columns=[\"names\"])\n", "df[\"names\"].str.title()" @@ -1259,26 +892,10 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "id": "d7a29663", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 [ada, lovelace]\n", - "1 [adam, smith]\n", - "2 [elinor, ostrom]\n", - "3 [grace, hopper]\n", - "4 [jean, bartik]\n", - "Name: names, dtype: object" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[\"names\"].str.split(\" \")" ] @@ -1293,79 +910,10 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "id": "85a5cd2c", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
01
0adalovelace
1adamsmith
2elinorostrom
3gracehopper
4jeanbartik
\n", - "
" - ], - "text/plain": [ - " 0 1\n", - "0 ada lovelace\n", - "1 adam smith\n", - "2 elinor ostrom\n", - "3 grace hopper\n", - "4 jean bartik" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[\"names\"].str.split(\" \", n=2, expand=True)" ] @@ -1384,36 +932,10 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "id": "2e8781ba", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "<>:1: SyntaxWarning: invalid escape sequence '\\w'\n", - "<>:1: SyntaxWarning: invalid escape sequence '\\w'\n", - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_60781/3354940721.py:1: SyntaxWarning: invalid escape sequence '\\w'\n", - " df[\"names\"].str.extract(\"(\\w+)\", expand=False)\n" - ] - }, - { - "data": { - "text/plain": [ - "0 ada\n", - "1 adam\n", - "2 elinor\n", - "3 grace\n", - "4 jean\n", - "Name: names, dtype: string" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[\"names\"].str.extract(\"(\\w+)\", expand=False)" ] @@ -1445,26 +967,10 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "id": "ba13d894", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 lovelace\n", - "1 smith\n", - "2 ostrom\n", - "3 hopper\n", - "4 bartik\n", - "Name: names, dtype: object" - ] - }, - "execution_count": 40, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[\"names\"].str.split().str.get(-1)" ] @@ -1479,79 +985,10 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "id": "056147d6", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
namestags
0ada lovelaceuk; cs
1adam smithuk; econ
2elinor ostromusa; econ
3grace hopperusa; cs
4jean bartikusa; cs
\n", - "
" - ], - "text/plain": [ - " names tags\n", - "0 ada lovelace uk; cs\n", - "1 adam smith uk; econ\n", - "2 elinor ostrom usa; econ\n", - "3 grace hopper usa; cs\n", - "4 jean bartik usa; cs" - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = pd.DataFrame(\n", " {\n", @@ -1578,91 +1015,10 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "id": "a5cbc10f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
cseconukusa
01010
10110
20101
31001
41001
\n", - "
" - ], - "text/plain": [ - " cs econ uk usa\n", - "0 1 0 1 0\n", - "1 0 1 1 0\n", - "2 0 1 0 1\n", - "3 1 0 0 1\n", - "4 1 0 0 1" - ] - }, - "execution_count": 42, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df[\"tags\"].str.get_dummies(\";\")" ] diff --git a/vis-layers.ipynb b/vis-layers.ipynb index 062b285..6a92e84 100644 --- a/vis-layers.ipynb +++ b/vis-layers.ipynb @@ -42,51 +42,10 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "a86fb211", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "import pandas as pd\n", "from lets_plot import *\n", @@ -108,156 +67,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "39a6d993", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
manufacturermodeldisplyearcyltransdrvctyhwyflclass
rownames
1audia41.819994auto(l5)f18.029.0pcompact
2audia41.819994manual(m5)f21.029.0pcompact
3audia42.020084manual(m6)f20.031.0pcompact
4audia42.020084auto(av)f21.030.0pcompact
5audia42.819996auto(l5)f16.026.0pcompact
\n", - "
" - ], - "text/plain": [ - " manufacturer model displ year cyl trans drv cty hwy fl \\\n", - "rownames \n", - "1 audi a4 1.8 1999 4 auto(l5) f 18.0 29.0 p \n", - "2 audi a4 1.8 1999 4 manual(m5) f 21.0 29.0 p \n", - "3 audi a4 2.0 2008 4 manual(m6) f 20.0 31.0 p \n", - "4 audi a4 2.0 2008 4 auto(av) f 21.0 30.0 p \n", - "5 audi a4 2.8 1999 6 auto(l5) f 16.0 26.0 p \n", - "\n", - " class \n", - "rownames \n", - "1 compact \n", - "2 compact \n", - "3 compact \n", - "4 compact \n", - "5 compact " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "mpg = pd.read_csv(\n", " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/mpg.csv\", index_col=0\n", @@ -304,262 +117,20 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "fe77349a", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\", color=\"class\")) + geom_point())" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "e77b5640", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\", shape=\"class\")) + geom_point())" ] @@ -574,262 +145,20 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "ef221330", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\", size=\"class\")) + geom_point())" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "d042255e", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\", alpha=\"class\")) + geom_point())" ] @@ -852,130 +181,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "618edcb4", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point(color=\"blue\"))" ] @@ -1041,263 +250,20 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "277a4c0f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point(size=4))" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "07247ba9", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_smooth(method=\"loess\", size=2))" ] @@ -1332,207 +298,20 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "4b20c825", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\", line=\"drv\")) + geom_smooth(method=\"loess\"))" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "84df3e78", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\", linetype=\"drv\")) + geom_smooth(method=\"loess\"))" ] @@ -1551,146 +330,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "c9e8d92f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\", color=\"drv\"))\n", @@ -1719,143 +362,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "b3916558", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point(aes(color=\"class\")) + geom_smooth())" ] @@ -1872,222 +382,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "38870eb5", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -2144,149 +442,14 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "ae75c5c1", "metadata": { "tags": [ "remove-cell" ] }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\", color=\"drv\"))\n", @@ -2307,136 +470,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "cb651300", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point() + facet_wrap(\"cyl\"))" ] @@ -2451,138 +488,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "61481052", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point() + facet_grid(\"drv\", \"cyl\"))" ] @@ -2599,139 +508,10 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "adcd9079", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", @@ -2742,137 +522,10 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "ceb2a354", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg) + geom_point(aes(x=\"displ\", y=\"hwy\")) + facet_wrap(\"class\", nrow=2))" ] @@ -2962,141 +615,10 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "f379e31b", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
caratcutcolorclaritydepthtablepricexyz
rownames
10.23IdealESI261.555.03263.953.982.43
20.21PremiumESI159.861.03263.893.842.31
30.23GoodEVS156.965.03274.054.072.31
40.29PremiumIVS262.458.03344.204.232.63
50.31GoodJSI263.358.03354.344.352.75
\n", - "
" - ], - "text/plain": [ - " carat cut color clarity depth table price x y z\n", - "rownames \n", - "1 0.23 Ideal E SI2 61.5 55.0 326 3.95 3.98 2.43\n", - "2 0.21 Premium E SI1 59.8 61.0 326 3.89 3.84 2.31\n", - "3 0.23 Good E VS1 56.9 65.0 327 4.05 4.07 2.31\n", - "4 0.29 Premium I VS2 62.4 58.0 334 4.20 4.23 2.63\n", - "5 0.31 Good J SI2 63.3 58.0 335 4.34 4.35 2.75" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "diamonds = pd.read_csv(\n", " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/diamonds.csv\",\n", @@ -3111,130 +633,10 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "d8faf1ab", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(diamonds, aes(x=\"cut\")) + geom_bar())" ] @@ -3277,108 +679,10 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "ca772dd5", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(\n", " ggplot(\n", @@ -3402,258 +706,20 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "f8da7d91", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"drv\", color=\"drv\")) + geom_bar())" ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "088e7550", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"drv\", fill=\"drv\")) + geom_bar())" ] @@ -3669,130 +735,10 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "181c70d2", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"drv\", fill=\"class\")) + geom_bar())" ] @@ -3812,132 +758,10 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "id": "a8e9c378", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"drv\", fill=\"class\")) + geom_bar(alpha=0.5, position=\"identity\"))" ] @@ -3955,131 +779,10 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "14205000", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"drv\", fill=\"class\")) + geom_bar(position=\"fill\"))" ] @@ -4095,131 +798,10 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "c33c4a03", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"drv\", fill=\"class\")) + geom_bar(position=\"dodge\"))" ] @@ -4236,129 +818,10 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "ba4161de", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point())" ] @@ -4380,130 +843,10 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "id": "414ce7af", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"displ\", y=\"hwy\")) + geom_point(position=\"jitter\"))" ] @@ -4535,129 +878,10 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "id": "9bc38aef", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "(ggplot(mpg, aes(x=\"cty\", y=\"hwy\")) + geom_point())" ] diff --git a/webscraping-and-apis.ipynb b/webscraping-and-apis.ipynb index 4bd60c2..802151d 100644 --- a/webscraping-and-apis.ipynb +++ b/webscraping-and-apis.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "51a55374", "metadata": { "tags": [ @@ -74,7 +74,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "a7f62293", "metadata": {}, "outputs": [], @@ -101,48 +101,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "06108a4d", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (10, 14)
rownamesnameyearmonthdayhourlatlongstatuscategorywindpressuretropicalstorm_force_diameterhurricane_force_diameter
i64stri64i64i64i64f64f64stri64i64i64strstr
1"Amy"1975627027.5-79.0"tropical depression"null251013nullnull
2"Amy"1975627628.5-79.0"tropical depression"null251013nullnull
3"Amy"19756271229.5-79.0"tropical depression"null251013nullnull
4"Amy"19756271830.5-79.0"tropical depression"null251013nullnull
5"Amy"1975628031.5-78.8"tropical depression"null251012nullnull
6"Amy"1975628632.4-78.7"tropical depression"null251012nullnull
7"Amy"19756281233.3-78.0"tropical depression"null251011nullnull
8"Amy"19756281834.0-77.0"tropical depression"null301006nullnull
9"Amy"1975629034.4-75.8"tropical storm"null351004nullnull
10"Amy"1975629634.0-74.8"tropical storm"null401002nullnull
" - ], - "text/plain": [ - "shape: (10, 14)\n", - "┌──────────┬──────┬──────┬───────┬───┬──────┬──────────┬─────────────────────┬─────────────────────┐\n", - "│ rownames ┆ name ┆ year ┆ month ┆ … ┆ wind ┆ pressure ┆ tropicalstorm_force ┆ hurricane_force_dia │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ _diameter ┆ meter │\n", - "│ i64 ┆ str ┆ i64 ┆ i64 ┆ ┆ i64 ┆ i64 ┆ --- ┆ --- │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ str ┆ str │\n", - "╞══════════╪══════╪══════╪═══════╪═══╪══════╪══════════╪═════════════════════╪═════════════════════╡\n", - "│ 1 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 25 ┆ 1013 ┆ null ┆ null │\n", - "│ 2 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 25 ┆ 1013 ┆ null ┆ null │\n", - "│ 3 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 25 ┆ 1013 ┆ null ┆ null │\n", - "│ 4 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 25 ┆ 1013 ┆ null ┆ null │\n", - "│ 5 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 25 ┆ 1012 ┆ null ┆ null │\n", - "│ 6 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 25 ┆ 1012 ┆ null ┆ null │\n", - "│ 7 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 25 ┆ 1011 ┆ null ┆ null │\n", - "│ 8 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 30 ┆ 1006 ┆ null ┆ null │\n", - "│ 9 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 35 ┆ 1004 ┆ null ┆ null │\n", - "│ 10 ┆ Amy ┆ 1975 ┆ 6 ┆ … ┆ 40 ┆ 1002 ┆ null ┆ null │\n", - "└──────────┴──────┴──────┴───────┴───┴──────┴──────────┴─────────────────────┴─────────────────────┘" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pl.read_csv(\n", " \"https://vincentarelbundock.github.io/Rdatasets/csv/dplyr/storms.csv\", n_rows=10\n", @@ -174,113 +136,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "6107093c", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "url = \"https://api.beta.ons.gov.uk/v1/data?uri=/employmentandlabourmarket/peopleinwork/employmentandemployeetypes/timeseries/jp9z/lms/previous/v108\"\n", "\n", @@ -343,60 +202,12 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "380ca743", "metadata": { "lines_to_next_cell": 2 }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/j1/0d3dmg7d6gnbc_zwrh8t9gcw0000gn/T/ipykernel_61971/1219974464.py:18: MapWithoutReturnDtypeWarning: Calling `map_elements` without specifying `return_dtype` can lead to unpredictable results. Specify `return_dtype` to silence this warning.\n", - " .with_columns(pl.col(\"country\").map_elements(lambda x: textwrap.fill(x, 10)))\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "shape: (5, 3)
economycountryEN.GHG.ALL.PC.CE.AR5
strstrf64
"USA""United\n", - "States"18.921098
"CHN""China"9.982534
"ECS""Europe &\n", - "Central\n", - "Asia"9.171446
"EAS""East Asia\n", - "& Pacific"8.435052
"IND""India"2.621464
" - ], - "text/plain": [ - "shape: (5, 3)\n", - "┌─────────┬───────────┬──────────────────────┐\n", - "│ economy ┆ country ┆ EN.GHG.ALL.PC.CE.AR5 │\n", - "│ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ f64 │\n", - "╞═════════╪═══════════╪══════════════════════╡\n", - "│ USA ┆ United ┆ 18.921098 │\n", - "│ ┆ States ┆ │\n", - "│ CHN ┆ China ┆ 9.982534 │\n", - "│ ECS ┆ Europe & ┆ 9.171446 │\n", - "│ ┆ Central ┆ │\n", - "│ ┆ Asia ┆ │\n", - "│ EAS ┆ East Asia ┆ 8.435052 │\n", - "│ ┆ & Pacific ┆ │\n", - "│ IND ┆ India ┆ 2.621464 │\n", - "└─────────┴───────────┴──────────────────────┘" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# World Bank CO2 equivalent emissions (metric tons per capita)\n", "# https://data.worldbank.org/indicator/EN.GHG.ALL.PC.CE.AR5\n", @@ -424,170 +235,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "00379b93", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - " " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "lp.LetsPlot.setup_html()\n", "\n", @@ -735,21 +386,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "073d89e3", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'\\n\\n\\n\\n\\n\\n\\n\\n\\n'" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "url = \"http://aeturrell.com/research\"\n", "page = requests.get(url)\n", @@ -769,35 +409,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "22f96be0", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - ">\n", - "
\n", - "
\n", - " \n", - " data science\n", - " \n", - "
\n", - "
\n", - " \n", - " public policy\n", - " \n", - "
\n", - "
\n", - " \n", - " complexity science\n", - " \n", - "
\n", - " <\n" - ] - } - ], + "outputs": [], "source": [ "soup = BeautifulSoup(page.text, \"html.parser\")\n", "print(soup.prettify()[60000:60500])" @@ -814,21 +429,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "d82775de", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "

Blundell, Jack, Emma Duchini, Stefania Simion, and Arthur Turrell. \"Pay transparency and gender equality.\" American Economic Journal: Economic Policy (2024). doi: 10.1257/pol.20220766

" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Get all paragraphs\n", "all_paras = soup.find_all(\"p\")\n", @@ -847,21 +451,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "11321154", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Blundell, Jack, Emma Duchini, Stefania Simion, and Arthur Turrell. \"Pay transparency and gender equality.\" American Economic Journal: Economic Policy (2024). doi: 10.1257/pol.20220766'" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "all_paras[1].text" ] @@ -877,40 +470,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "feac1fdd", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['Blundell, Jack, Emma Duchini, Stefania Simion, and Arthur Turrell. \"Pay transparency and gender equality.\" American Economic Journal: Economic Policy (2024). doi: 10.1257/pol.20220766',\n", - " 'Botta, Federico, Robin Lovelace, Laura Gilbert, and Arthur Turrell. \"Packaging code and data for reproducible research: A case study of journey time statistics.\" Environment and Planning B: Urban Analytics and City Science (2024): 23998083241267331. doi: 10.1177/23998083241267331',\n", - " 'Kalamara, Eleni, Arthur Turrell, Chris Redl, George Kapetanios, and Sujit Kapadia. \"Making text count: economic forecasting using newspaper text.\" Journal of Applied Econometrics 37, no. 5 (2022): 896-919. doi: 10.1002/jae.2907',\n", - " 'Turrell, A., Speigner, B., Copple, D., Djumalieva, J. and Thurgood, J., 2021. Is the UK’s productivity puzzle mostly driven by occupational mismatch? An analysis using big data on job vacancies. Labour Economics, 71, p.102013. doi: 10.1016/j.labeco.2021.102013',\n", - " 'Haldane, Andrew G., and Arthur E. Turrell. \"Drawing on different disciplines: macroeconomic agent-based models.\" Journal of Evolutionary Economics 29 (2019): 39-66. doi: 10.1007/s00191-018-0557-5',\n", - " 'Haldane, Andrew G., and Arthur E. Turrell. \"An interdisciplinary model for macroeconomics.\" Oxford Review of Economic Policy 34, no. 1-2 (2018): 219-251. doi: 10.1093/oxrep/grx051',\n", - " 'Braun-Munzinger, Karen, Z. Liu, and A. E. Turrell. \"An agent-based model of corporate bond trading.\" Quantitative Finance 18, no. 4 (2018): 591-608. doi: 10.1080/14697688.2017.1380310',\n", - " 'Turrell, A. E., M. Sherlock, and S. J. Rose. \"Efficient evaluation of collisional energy transfer terms for plasma particle simulations.\" Journal of Plasma Physics 82, no. 1 (2016): 905820107. doi: 10.1017/S0022377816000131',\n", - " 'Turrell, A. E., M. Sherlock, and S. J. Rose. \"Ultrafast collisional ion heating by electrostatic shocks.\" Nature Communications 6, no. 1 (2015): 8905. doi: 10.1038/ncomms9905',\n", - " 'Turrell, Arthur E., Mark Sherlock, and Steven J. Rose. \"Self-consistent inclusion of classical large-angle Coulomb collisions in plasma Monte Carlo simulations.\" Journal of Computational Physics 299 (2015): 144-155. doi: 10.1016/j.jcp.2015.06.034',\n", - " 'Turrell, Arthur E., Mark Sherlock, and Steven J. Rose. \"A Monte Carlo algorithm for degenerate plasmas.\" Journal of Computational Physics 249 (2013): 13-21. doi: 10.1016/j.jcp.2013.03.052',\n", - " 'Turrell, Arthur. \"Cutting through Complexity: How Data Science Can Help Policymakers Understand the World.\" In The Economy as a Complex Evolving System, Part IV. Sante Fe Institute, 2025. doi: https://doi.org/10.37911/9781947864665.11',\n", - " 'Duchini, Emma, Stefania Simion, and Arthur Turrell. \"A Review of the Effects of Pay Transparency.\" In Oxford Research Encyclopedia of Economics and Finance, Oxford University Press, 2024. doi: 10.1093/acrefore/9780190625979.013.860',\n", - " 'Turrell, Arthur, Bradley Speigner, Jyldyz Djumalieva, David Copple, and James Thurgood. \"6. Transforming Naturally Occurring Text Data into Economic Statistics.\" In Big Data for Twenty-First-Century Economic Statistics, pp. 173-208. University of Chicago Press, 2022. doi: 10.7208/chicago/9780226801391-008',\n", - " 'Turrell, Arthur. \"Agent-based models: understanding the economy from the bottom up\" In Quarterly Bulletin, Q4. Bank of England, 2016.',\n", - " 'Cohen, Samuel N., Giulia Mantoan, Lars Nesheim, Áureo de Paula, Arthur Turrell, and Lingyi Yang. Nowcasting using regression on signatures arXiv preprint arXiv:2305.10256v2 (2025).',\n", - " 'Van Dijcke, David, Marcus Buckmann, Arthur Turrell, and Tomas Key. \"Vacancy Posting, Firm Balance Sheets, and Pandemic Policy Interventions.\" Bank of England Staff Working Paper Series 1033 (2022).',\n", - " 'Draca, Mirko, Emma Duchini, Roland Rathelot, Arthur Turrell, and Giulia Vattuone. Revolution in Progress? The Rise of Remote Work in the UK. University of Warwick, Department of Economics, 2022.',\n", - " 'Hill, Edward, Marco Bardoscia, and Arthur Turrell. \"Solving heterogeneous general equilibrium economic models with deep reinforcement learning.\" arXiv arXiv:2103.16977 (2021).',\n", - " 'Turrell, Arthur, James Thurgood, David Copple, Jyldyz Djumalieva, and Bradley Speigner. \"Using online job vacancies to understand the UK labour market from the bottom-up.\" Bank of England Staff Working Papers 742 (2018).']" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "projects = soup.find_all(\"div\", class_=\"project-content listing-pub-info\")\n", "projects = [x.text.strip() for x in projects]\n", @@ -955,27 +518,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "0ada9ce7", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "shape: (3, 4)\n", - "┌─────┬────────────┬───────────┬──────────┐\n", - "│ # ┆ First Name ┆ Last Name ┆ Username │\n", - "│ --- ┆ --- ┆ --- ┆ --- │\n", - "│ i64 ┆ str ┆ str ┆ str │\n", - "╞═════╪════════════╪═══════════╪══════════╡\n", - "│ 1 ┆ Mark ┆ Otto ┆ @mdo │\n", - "│ 2 ┆ Jacob ┆ Thornton ┆ @fat │\n", - "│ 3 ┆ Larry ┆ the Bird ┆ @twitter │\n", - "└─────┴────────────┴───────────┴──────────┘\n" - ] - } - ], + "outputs": [], "source": [ "import polars as pl\n", "\n", diff --git a/whole-game.ipynb b/whole-game.ipynb index 5498d0e..c0b6e5f 100644 --- a/whole-game.ipynb +++ b/whole-game.ipynb @@ -13,7 +13,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "51a55374", "metadata": { "tags": [ @@ -29,116 +29,14 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "209ef434", "metadata": { "tags": [ "remove-input" ] }, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "cluster_0\n", - "\n", - "Understand\n", - "\n", - "\n", - "\n", - "Import\n", - "\n", - "Import\n", - "\n", - "\n", - "\n", - "Clean\n", - "\n", - "Clean\n", - "\n", - "\n", - "\n", - "Import->Clean\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Analyse\n", - "\n", - "Analyse\n", - "\n", - "\n", - "\n", - "Clean->Analyse\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Visualise\n", - "\n", - "Visualise\n", - "\n", - "\n", - "\n", - "Visualise->Analyse\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Transform\n", - "\n", - "Transform\n", - "\n", - "\n", - "\n", - "Analyse->Transform\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Communicate\n", - "\n", - "Communicate\n", - "\n", - "\n", - "\n", - "Analyse->Communicate\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "Transform->Visualise\n", - "\n", - "\n", - "\n", - "\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# | echo: false\n", "import graphviz\n", diff --git a/workflow-basics.ipynb b/workflow-basics.ipynb index 27b2168..64621de 100644 --- a/workflow-basics.ipynb +++ b/workflow-basics.ipynb @@ -26,19 +26,10 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "23465996", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.15\n", - "44.666666666666664\n" - ] - } - ], + "outputs": [], "source": [ "print(1 / 200 * 30)\n", "print((59 + 73 + 2) / 3)" @@ -55,18 +46,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "bdd3c2c0", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.0\n" - ] - } - ], + "outputs": [], "source": [ "import numpy as np\n", "\n", @@ -84,18 +67,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "cead1264", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "12\n" - ] - } - ], + "outputs": [], "source": [ "x = 3 * 4\n", "print(x)" @@ -112,18 +87,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "91a44d83", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[1, 2, 3, 5, 7, 11, 13]\n" - ] - } - ], + "outputs": [], "source": [ "primes = [1, 2, 3, 5, 7, 11, 13]\n", "print(primes)" @@ -140,21 +107,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "1a526124", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[3, 6, 9, 15, 21, 33, 39]" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "[element * 3 for element in primes]" ] @@ -198,21 +154,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "c061f9cc", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[2, 4, 6, 10, 14, 22, 26]" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# define primes\n", "primes = [1, 2, 3, 5, 7, 11, 13]\n", @@ -248,21 +193,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "e77540e0", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[1, 2, 3, 5, 7, 11, 13]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "primes" ] @@ -278,21 +212,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "9dd0fd49", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "list" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "type(primes)" ] @@ -350,21 +273,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "a5998cb5", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[1, 2, 3, 5, 7, 11, 13]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "primes" ] @@ -379,7 +291,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "ae55d78e", "metadata": {}, "outputs": [], @@ -401,7 +313,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "038c7d52", "metadata": {}, "outputs": [], @@ -457,21 +369,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "97af119d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "42" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "sum(primes)" ] @@ -487,21 +388,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "14ce9b99", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "52" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "sum(primes, start=10)" ] @@ -517,26 +407,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "f5e45616", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Help on built-in function sum in module builtins:\n", - "\n", - "sum(iterable, /, start=0)\n", - " Return the sum of a 'start' value (default: 0) plus an iterable of numbers\n", - "\n", - " When the iterable is empty, return the start value.\n", - " This function is intended specifically for use with numeric values and may\n", - " reject non-numeric types.\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "help(sum)" ] diff --git a/workflow-style.ipynb b/workflow-style.ipynb index 7a1a2a4..3de114b 100644 --- a/workflow-style.ipynb +++ b/workflow-style.ipynb @@ -148,40 +148,10 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "f0f5bb37", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (3, 4)
col3col4col1col2
strstru32f64
"a""alpha"10.0
"b""gamma"20.0
"a""gamma"10.0
" - ], - "text/plain": [ - "shape: (3, 4)\n", - "┌──────┬───────┬──────┬──────┐\n", - "│ col3 ┆ col4 ┆ col1 ┆ col2 │\n", - "│ --- ┆ --- ┆ --- ┆ --- │\n", - "│ str ┆ str ┆ u32 ┆ f64 │\n", - "╞══════╪═══════╪══════╪══════╡\n", - "│ a ┆ alpha ┆ 1 ┆ 0.0 │\n", - "│ b ┆ gamma ┆ 2 ┆ 0.0 │\n", - "│ a ┆ gamma ┆ 1 ┆ 0.0 │\n", - "└──────┴───────┴──────┴──────┘" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import polars as pl\n", "\n", From 03e70f993c41b4a953156f45669e8c10900f740c Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 12 Jul 2026 05:00:56 +0100 Subject: [PATCH 33/34] refactor: replace pandas with polars in vis-layers.ipynb data processing examples --- vis-layers.ipynb | 205 +++++++++++++++++++++++------------------------ 1 file changed, 101 insertions(+), 104 deletions(-) diff --git a/vis-layers.ipynb b/vis-layers.ipynb index 6a92e84..1807a29 100644 --- a/vis-layers.ipynb +++ b/vis-layers.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "95f0a171", + "id": "0", "metadata": {}, "source": [ "# Layers {#sec-vis-layers}\n" @@ -10,7 +10,7 @@ }, { "cell_type": "markdown", - "id": "9fdd3b8a", + "id": "1", "metadata": {}, "source": [ "## Introduction\n", @@ -30,12 +30,12 @@ }, { "cell_type": "markdown", - "id": "17575f3a", + "id": "2", "metadata": {}, "source": [ "### Prerequisites\n", "\n", - "You will need to install the **letsplot** package for this chapter, as well as **pandas**.\n", + "You will need to install the **letsplot** package for this chapter, as well as **polars**.\n", "\n", "In your Python session, import the libraries we'll be using:" ] @@ -43,11 +43,12 @@ { "cell_type": "code", "execution_count": null, - "id": "a86fb211", + "id": "3", "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "\n", + "import polars as pl\n", "from lets_plot import *\n", "\n", "LetsPlot.setup_html()" @@ -55,7 +56,7 @@ }, { "cell_type": "markdown", - "id": "55b00fde", + "id": "4", "metadata": {}, "source": [ "## Aesthetic mappings\n", @@ -68,35 +69,29 @@ { "cell_type": "code", "execution_count": null, - "id": "39a6d993", + "id": "5", "metadata": {}, "outputs": [], "source": [ - "mpg = pd.read_csv(\n", - " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/mpg.csv\", index_col=0\n", + "mpg = pl.read_csv(\n", + " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/mpg.csv\"\n", ")\n", "\n", - "mpg = mpg.astype(\n", - " {\n", - " \"manufacturer\": \"category\",\n", - " \"model\": \"category\",\n", - " \"displ\": \"double\",\n", - " \"year\": \"int64\",\n", - " \"cyl\": \"int64\",\n", - " \"trans\": \"category\",\n", - " \"drv\": \"category\",\n", - " \"cty\": \"double\",\n", - " \"hwy\": \"double\",\n", - " \"fl\": \"category\",\n", - " \"class\": \"category\",\n", - " }\n", - ")\n", + "mpg = mpg.with_columns([\n", + " pl.col(\"manufacturer\").cast(pl.Categorical),\n", + " pl.col(\"model\").cast(pl.Categorical),\n", + " pl.col(\"trans\").cast(pl.Categorical),\n", + " pl.col(\"drv\").cast(pl.Categorical),\n", + " pl.col(\"fl\").cast(pl.Categorical),\n", + " pl.col(\"class\").cast(pl.Categorical),\n", + "])\n", + "\n", "mpg.head()" ] }, { "cell_type": "markdown", - "id": "6d6f1307", + "id": "6", "metadata": {}, "source": [ "Among the variables in `mpg` are:\n", @@ -118,7 +113,7 @@ { "cell_type": "code", "execution_count": null, - "id": "fe77349a", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -128,7 +123,7 @@ { "cell_type": "code", "execution_count": null, - "id": "e77b5640", + "id": "8", "metadata": {}, "outputs": [], "source": [ @@ -137,7 +132,7 @@ }, { "cell_type": "markdown", - "id": "53e51510", + "id": "9", "metadata": {}, "source": [ "Similarly, we can map `class` to `size` or `alpha` aesthetics as well, which control the shape and the transparency of the points, respectively." @@ -146,7 +141,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ef221330", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -156,7 +151,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d042255e", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -165,7 +160,7 @@ }, { "cell_type": "markdown", - "id": "ec07349f", + "id": "12", "metadata": {}, "source": [ "While we are able to do it, mapping an unordered discrete (categorical) variable (`class`) to an ordered aesthetic variable (`size` or `alpha`) is generally not a good idea because it implies a ranking that does not in fact exist.\n", @@ -182,7 +177,7 @@ { "cell_type": "code", "execution_count": null, - "id": "618edcb4", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -191,7 +186,7 @@ }, { "cell_type": "markdown", - "id": "52611640", + "id": "14", "metadata": {}, "source": [ "Here, the colour doesn't convey information about a variable, but only changes the appearance of the plot.\n", @@ -206,7 +201,7 @@ }, { "cell_type": "markdown", - "id": "37d49ff8", + "id": "15", "metadata": {}, "source": [ "So far we have discussed aesthetics that we can map or set in a scatterplot, when using a point geom.\n", @@ -217,7 +212,7 @@ }, { "cell_type": "markdown", - "id": "3f1da019", + "id": "16", "metadata": {}, "source": [ "1. Create a scatterplot of `hwy` vs. `displ` where the points are pink filled in triangles.\n", @@ -240,7 +235,7 @@ }, { "cell_type": "markdown", - "id": "83aa98f0", + "id": "17", "metadata": {}, "source": [ "## Geometric objects\n", @@ -251,7 +246,7 @@ { "cell_type": "code", "execution_count": null, - "id": "277a4c0f", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -261,7 +256,7 @@ { "cell_type": "code", "execution_count": null, - "id": "07247ba9", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -270,7 +265,7 @@ }, { "cell_type": "markdown", - "id": "58824a10", + "id": "20", "metadata": {}, "source": [ "Both plots contain the same x variable, the same y variable, and both describe the same data.\n", @@ -283,7 +278,7 @@ }, { "cell_type": "markdown", - "id": "0db26c6c", + "id": "21", "metadata": {}, "source": [ "Every geom function in **lets-plot** takes a `mapping` argument, either defined locally in the geom layer or globally in the `ggplot()` layer.\n", @@ -299,7 +294,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4b20c825", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -309,7 +304,7 @@ { "cell_type": "code", "execution_count": null, - "id": "84df3e78", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -318,7 +313,7 @@ }, { "cell_type": "markdown", - "id": "7b114911", + "id": "24", "metadata": {}, "source": [ "Here, `geom_smooth()` separates the cars into three lines based on their `drv` value, which describes a car's drive train.\n", @@ -331,7 +326,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c9e8d92f", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -344,7 +339,7 @@ }, { "cell_type": "markdown", - "id": "b6392da1", + "id": "26", "metadata": {}, "source": [ "Notice that this plot contains two geoms in the same graph.\n", @@ -363,7 +358,7 @@ { "cell_type": "code", "execution_count": null, - "id": "b3916558", + "id": "27", "metadata": {}, "outputs": [], "source": [ @@ -372,7 +367,7 @@ }, { "cell_type": "markdown", - "id": "88708546", + "id": "28", "metadata": {}, "source": [ "You can use the same idea to specify different data for each layer.\n", @@ -383,23 +378,23 @@ { "cell_type": "code", "execution_count": null, - "id": "38870eb5", + "id": "29", "metadata": {}, "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", " + geom_point()\n", - " + geom_point(data=mpg.loc[mpg[\"class\"] == \"2seater\", :], color=\"red\", size=2)\n", + " + geom_point(data=mpg.filter(pl.col(\"class\") == \"2seater\"), color=\"red\", size=2)\n", " + geom_point(\n", - " data=mpg.loc[mpg[\"class\"] == \"2seater\", :], shape=1, size=3, color=\"red\"\n", + " data=mpg.filter(pl.col(\"class\") == \"2seater\"), shape=1, size=3, color=\"red\"\n", " )\n", ")" ] }, { "cell_type": "markdown", - "id": "35a3d017", + "id": "30", "metadata": {}, "source": [ "Geoms are the fundamental building blocks of **lets-plot**.\n", @@ -415,7 +410,7 @@ }, { "cell_type": "markdown", - "id": "39a12f36", + "id": "31", "metadata": {}, "source": [ "### Exercises\n", @@ -443,7 +438,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ae75c5c1", + "id": "32", "metadata": { "tags": [ "remove-cell" @@ -460,7 +455,7 @@ }, { "cell_type": "markdown", - "id": "4c1d45ab", + "id": "33", "metadata": {}, "source": [ "## Facets\n", @@ -471,7 +466,7 @@ { "cell_type": "code", "execution_count": null, - "id": "cb651300", + "id": "34", "metadata": {}, "outputs": [], "source": [ @@ -480,7 +475,7 @@ }, { "cell_type": "markdown", - "id": "0cd9ef67", + "id": "35", "metadata": {}, "source": [ "To facet your plot with the combination of two variables, switch from `facet_wrap()` to `facet_grid()`." @@ -489,7 +484,7 @@ { "cell_type": "code", "execution_count": null, - "id": "61481052", + "id": "36", "metadata": {}, "outputs": [], "source": [ @@ -498,7 +493,7 @@ }, { "cell_type": "markdown", - "id": "4f502141", + "id": "37", "metadata": {}, "source": [ "By default each of the facets share the same scale and range for x and y axes.\n", @@ -509,7 +504,7 @@ { "cell_type": "code", "execution_count": null, - "id": "adcd9079", + "id": "38", "metadata": {}, "outputs": [], "source": [ @@ -523,7 +518,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ceb2a354", + "id": "39", "metadata": {}, "outputs": [], "source": [ @@ -532,7 +527,7 @@ }, { "cell_type": "markdown", - "id": "5e3a949f", + "id": "40", "metadata": {}, "source": [ "### Exercises\n", @@ -602,7 +597,7 @@ }, { "cell_type": "markdown", - "id": "cacf1fb5", + "id": "41", "metadata": {}, "source": [ "## Statistical transformations\n", @@ -616,34 +611,36 @@ { "cell_type": "code", "execution_count": null, - "id": "f379e31b", + "id": "42", "metadata": {}, "outputs": [], "source": [ - "diamonds = pd.read_csv(\n", - " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/diamonds.csv\",\n", - " index_col=0,\n", + "diamonds = pl.read_csv(\n", + " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/diamonds.csv\"\n", ")\n", "diamonds_cut_order = [\"Fair\", \"Good\", \"Very Good\", \"Premium\", \"Ideal\"]\n", - "diamonds[\"cut\"] = diamonds[\"cut\"].astype(\n", - " pd.CategoricalDtype(categories=diamonds_cut_order, ordered=True)\n", + "diamonds = diamonds.with_columns(\n", + " pl.col(\"cut\").cast(pl.Enum(diamonds_cut_order))\n", ")\n", + "\n", "diamonds.head()" ] }, { "cell_type": "code", "execution_count": null, - "id": "d8faf1ab", + "id": "43", "metadata": {}, "outputs": [], "source": [ - "(ggplot(diamonds, aes(x=\"cut\")) + geom_bar())" + "diamonds_sorted = diamonds.sort(\"cut\")\n", + "\n", + "(ggplot(diamonds_sorted, aes(x=\"cut\")) + geom_bar())" ] }, { "cell_type": "markdown", - "id": "a27666cf", + "id": "44", "metadata": {}, "source": [ "On the x-axis, the chart displays `cut`, a variable from `diamonds`.\n", @@ -666,7 +663,7 @@ }, { "cell_type": "markdown", - "id": "62519f73", + "id": "45", "metadata": {}, "source": [ "You can learn which stat a geom uses by inspecting the default value for the `stat` argument.\n", @@ -680,22 +677,25 @@ { "cell_type": "code", "execution_count": null, - "id": "ca772dd5", + "id": "46", "metadata": {}, "outputs": [], "source": [ + "diamonds_counts = (\n", + " diamonds.group_by(\"cut\")\n", + " .agg(pl.len().alias(\"counts\"))\n", + " .sort(\"cut\") \n", + ")\n", + "\n", "(\n", - " ggplot(\n", - " diamonds.value_counts(\"cut\").reset_index(name=\"counts\"),\n", - " aes(x=\"cut\", y=\"counts\"),\n", - " )\n", + " ggplot(diamonds_counts, aes(x=\"cut\", y=\"counts\"))\n", " + geom_bar(stat=\"identity\")\n", ")" ] }, { "cell_type": "markdown", - "id": "e365aaaf", + "id": "47", "metadata": {}, "source": [ "## Position adjustments\n", @@ -707,7 +707,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f8da7d91", + "id": "48", "metadata": {}, "outputs": [], "source": [ @@ -717,7 +717,7 @@ { "cell_type": "code", "execution_count": null, - "id": "088e7550", + "id": "49", "metadata": {}, "outputs": [], "source": [ @@ -726,7 +726,7 @@ }, { "cell_type": "markdown", - "id": "c3f7a0a3", + "id": "50", "metadata": {}, "source": [ "Note what happens if you map the fill aesthetic to another variable, like `class`: the bars are automatically stacked.\n", @@ -736,7 +736,7 @@ { "cell_type": "code", "execution_count": null, - "id": "181c70d2", + "id": "51", "metadata": {}, "outputs": [], "source": [ @@ -745,7 +745,7 @@ }, { "cell_type": "markdown", - "id": "b3f74621", + "id": "52", "metadata": {}, "source": [ "The stacking is performed automatically using the **position adjustment** specified by the `position` argument.\n", @@ -759,7 +759,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a8e9c378", + "id": "53", "metadata": {}, "outputs": [], "source": [ @@ -768,7 +768,7 @@ }, { "cell_type": "markdown", - "id": "2aeffbf7", + "id": "54", "metadata": {}, "source": [ "The identity position adjustment is more useful for 2d geoms, like points, where it is the default.\n", @@ -780,7 +780,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14205000", + "id": "55", "metadata": {}, "outputs": [], "source": [ @@ -789,7 +789,7 @@ }, { "cell_type": "markdown", - "id": "3a8e72bd", + "id": "56", "metadata": {}, "source": [ "- `position = \"dodge\"` places overlapping objects directly *beside* one another.\n", @@ -799,7 +799,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c33c4a03", + "id": "57", "metadata": {}, "outputs": [], "source": [ @@ -808,7 +808,7 @@ }, { "cell_type": "markdown", - "id": "cd46f22b", + "id": "58", "metadata": {}, "source": [ "There's one other type of adjustment that's not useful for bar charts, but can be very useful for scatterplots.\n", @@ -819,7 +819,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ba4161de", + "id": "59", "metadata": {}, "outputs": [], "source": [ @@ -828,7 +828,7 @@ }, { "cell_type": "markdown", - "id": "28a3bfc7", + "id": "60", "metadata": {}, "source": [ "The underlying values of `hwy` and `displ` are rounded so the points appear on a grid and many points overlap each other.\n", @@ -844,7 +844,7 @@ { "cell_type": "code", "execution_count": null, - "id": "414ce7af", + "id": "61", "metadata": {}, "outputs": [], "source": [ @@ -853,7 +853,7 @@ }, { "cell_type": "markdown", - "id": "96277926", + "id": "62", "metadata": {}, "source": [ "Adding randomness seems like a strange way to improve your plot, but while it makes your graph less accurate at small scales, it makes your graph *more* revealing at large scales.\n", @@ -867,7 +867,7 @@ }, { "cell_type": "markdown", - "id": "90d09736", + "id": "63", "metadata": {}, "source": [ "### Exercises\n", @@ -879,7 +879,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9bc38aef", + "id": "64", "metadata": {}, "outputs": [], "source": [ @@ -888,7 +888,7 @@ }, { "cell_type": "markdown", - "id": "57b6523f", + "id": "65", "metadata": {}, "source": [ "2. What, if anything, is the difference between the two plots?\n", @@ -908,7 +908,7 @@ }, { "cell_type": "markdown", - "id": "a3138f75", + "id": "66", "metadata": {}, "source": [ "3. What parameters to `geom_jitter()` control the amount of jittering?\n", @@ -919,7 +919,7 @@ }, { "cell_type": "markdown", - "id": "7d302c31", + "id": "67", "metadata": {}, "source": [ "## The layered grammar of graphics\n", @@ -954,7 +954,7 @@ }, { "cell_type": "markdown", - "id": "c0f7a7ff", + "id": "68", "metadata": {}, "source": [ "At this point, you would have a complete graph, but you could further adjust the positions of the geoms within the coordinate system (a position adjustment) or split the graph into subplots (faceting).\n", @@ -971,9 +971,6 @@ } ], "metadata": { - "interpreter": { - "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" - }, "jupytext": { "cell_metadata_filter": "-all", "encoding": "# -*- coding: utf-8 -*-", @@ -981,7 +978,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": ".venv", "language": "python", "name": "python3" }, From a1e1d0dbb66aef63bb21ee9061cf2960289098d4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 14 Jul 2026 22:35:02 +0100 Subject: [PATCH 34/34] style: run pre-commit on vis-layers notebook to format and clean code --- vis-layers.ipynb | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/vis-layers.ipynb b/vis-layers.ipynb index 1807a29..cf6a92d 100644 --- a/vis-layers.ipynb +++ b/vis-layers.ipynb @@ -47,7 +47,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "import polars as pl\n", "from lets_plot import *\n", "\n", @@ -73,19 +72,19 @@ "metadata": {}, "outputs": [], "source": [ - "mpg = pl.read_csv(\n", - " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/mpg.csv\"\n", + "mpg = pl.read_csv(\"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/mpg.csv\")\n", + "\n", + "mpg = mpg.with_columns(\n", + " [\n", + " pl.col(\"manufacturer\").cast(pl.Categorical),\n", + " pl.col(\"model\").cast(pl.Categorical),\n", + " pl.col(\"trans\").cast(pl.Categorical),\n", + " pl.col(\"drv\").cast(pl.Categorical),\n", + " pl.col(\"fl\").cast(pl.Categorical),\n", + " pl.col(\"class\").cast(pl.Categorical),\n", + " ]\n", ")\n", "\n", - "mpg = mpg.with_columns([\n", - " pl.col(\"manufacturer\").cast(pl.Categorical),\n", - " pl.col(\"model\").cast(pl.Categorical),\n", - " pl.col(\"trans\").cast(pl.Categorical),\n", - " pl.col(\"drv\").cast(pl.Categorical),\n", - " pl.col(\"fl\").cast(pl.Categorical),\n", - " pl.col(\"class\").cast(pl.Categorical),\n", - "])\n", - "\n", "mpg.head()" ] }, @@ -619,9 +618,7 @@ " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/diamonds.csv\"\n", ")\n", "diamonds_cut_order = [\"Fair\", \"Good\", \"Very Good\", \"Premium\", \"Ideal\"]\n", - "diamonds = diamonds.with_columns(\n", - " pl.col(\"cut\").cast(pl.Enum(diamonds_cut_order))\n", - ")\n", + "diamonds = diamonds.with_columns(pl.col(\"cut\").cast(pl.Enum(diamonds_cut_order)))\n", "\n", "diamonds.head()" ] @@ -681,16 +678,9 @@ "metadata": {}, "outputs": [], "source": [ - "diamonds_counts = (\n", - " diamonds.group_by(\"cut\")\n", - " .agg(pl.len().alias(\"counts\"))\n", - " .sort(\"cut\") \n", - ")\n", + "diamonds_counts = diamonds.group_by(\"cut\").agg(pl.len().alias(\"counts\")).sort(\"cut\")\n", "\n", - "(\n", - " ggplot(diamonds_counts, aes(x=\"cut\", y=\"counts\"))\n", - " + geom_bar(stat=\"identity\")\n", - ")" + "(ggplot(diamonds_counts, aes(x=\"cut\", y=\"counts\")) + geom_bar(stat=\"identity\"))" ] }, {