diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cac3d2d..ad7aae3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,59 +1,59 @@ name: tests on: - push: - branches: [main] - pull_request: + push: + branches: [main] + pull_request: concurrency: - group: tests-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: tests-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - uses: pre-commit/action@v3.0.1 - - build: - runs-on: ubuntu-latest - steps: - - name: Check out the repository - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - - name: Install uv - uses: astral-sh/setup-uv@v4 - - - name: Set up Python - run: uv python install - - - name: Install dependencies - run: uv sync --all-extras --dev - - - name: Install Quarto - uses: quarto-dev/quarto-actions/setup@v2 - with: - version: "1.5.57" - - - name: set timezone - run: | - TZ="Europe/London" && - sudo ln -snf /usr/share/zoneinfo/$TZ /etc/localtime - - - name: install linux deps - run: | - sudo apt-get -y install openssl graphviz nano texlive graphviz-dev unzip build-essential - - - name: build the book - run: | - uv run quarto render --execute - - - name: success - run: | - echo "Success in building book without errors!" + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - uses: pre-commit/action@v3.0.1 + + build: + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Set up Python + run: uv python install + + - name: Install dependencies + run: uv sync --all-extras --dev + + - name: Install Quarto + uses: quarto-dev/quarto-actions/setup@v2 + with: + version: "1.5.57" + + - name: set timezone + run: | + TZ="Europe/London" && + sudo ln -snf /usr/share/zoneinfo/$TZ /etc/localtime + + - name: install linux deps + run: | + sudo apt-get -y install openssl graphviz nano texlive graphviz-dev unzip build-essential + + - name: build the book + run: | + uv run quarto render --execute + + - name: success + run: | + echo "Success in building book without errors!" diff --git a/_quarto.yml b/_quarto.yml index 51699c0..5c5e7f4 100644 --- a/_quarto.yml +++ b/_quarto.yml @@ -7,7 +7,7 @@ book: author: "The Py4DS Community" favicon: favicon.ico cover-image: logo.png - repo-url: https://github.com/aeturrell/python4DS + repo-url: https://github.com/datathink/python4DSpolars repo-branch: main repo-actions: [edit, issue] google-analytics: "G-LXJC37BJVX" diff --git a/boolean-data.ipynb b/boolean-data.ipynb index 931bbb2..d389d25 100644 --- a/boolean-data.ipynb +++ b/boolean-data.ipynb @@ -714,7 +714,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/categorical-data.ipynb b/categorical-data.ipynb index 5796a26..ab6ba30 100644 --- a/categorical-data.ipynb +++ b/categorical-data.ipynb @@ -378,7 +378,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/command-line.md b/command-line.md index 1079842..f9eeb20 100644 --- a/command-line.md +++ b/command-line.md @@ -1,6 +1,6 @@ # The Command Line {#sec-command-line} -In this chapter, you'll meet the *command line* and learn how to use it. Beyond a few key commands like `uv add ` you don't strictly need to know how to use the command line to follow the rest of this book. However, even a tiny bit of knowledge of the command line goes a long way in coding and will serve you well. +In this chapter, you'll meet the _command line_ and learn how to use it. Beyond a few key commands like `uv add ` you don't strictly need to know how to use the command line to follow the rest of this book. However, even a tiny bit of knowledge of the command line goes a long way in coding and will serve you well. To try out any of the commands in this chapter on your machine, you can select 'New Terminal' from the menu bar in Visual Studio Code (Mac and Linux), use the Windows Subsystem for Linux or git bash (Windows), or use a free [online terminal](https://cocalc.com/doc/terminal.html). @@ -8,17 +8,17 @@ This chapter has benefited from numerous sources, including absolutely excellent ## What is the command line? -The command line is a way to directly issue text-based commands to a computer one line at a time (as distinct from a graphical user interface, or GUI, that you navigate with a mouse). It goes under many names: shell, bash, terminal, CLI, and command line. These are actually different things but most people tend to use them to mean the same thing most of the time. The *shell* is the part of an operating system that you interact with but mostly people use shell to mean the command line. *bash* is the programming language that is used in the command line; it's actually a synonym for 'Born Again SHell'. The *terminal* is sometimes used to refer to the command line on Macs. Finally, a *CLI* is just an acronym for command line interface, and is often used in the context of an application; for example, uv has a command line interface because you run it on the command line to install packages (`uv add packagename`). +The command line is a way to directly issue text-based commands to a computer one line at a time (as distinct from a graphical user interface, or GUI, that you navigate with a mouse). It goes under many names: shell, bash, terminal, CLI, and command line. These are actually different things but most people tend to use them to mean the same thing most of the time. The _shell_ is the part of an operating system that you interact with but mostly people use shell to mean the command line. _bash_ is the programming language that is used in the command line; it's actually a synonym for 'Born Again SHell'. The _terminal_ is sometimes used to refer to the command line on Macs. Finally, a _CLI_ is just an acronym for command line interface, and is often used in the context of an application; for example, uv has a command line interface because you run it on the command line to install packages (`uv add packagename`). It's worth mentioning that there's a big difference between the command line on UNIX based systems (MacOS and Linux), and on Windows systems. Here, we'll only address the UNIX version. There is a command line on Windows but it's not widely used for coding. If you're on a Windows machine, you can access a UNIX command line using the Windows Subsystem for Linux. ## Why is the command line useful? -The command line has many uses. Graphical user interfaces are, generally, a bit easier to use *but* they're not very repeatable or scalable. Because the command line uses text-based instructions and can be programmed, it is both repeatable and scalable; properties that are very useful for research and analysis. +The command line has many uses. Graphical user interfaces are, generally, a bit easier to use _but_ they're not very repeatable or scalable. Because the command line uses text-based instructions and can be programmed, it is both repeatable and scalable; properties that are very useful for research and analysis. The broad reasons you might use the command line to issue instructions include: -- software functionality: some software *only* has a command line interface +- software functionality: some software _only_ has a command line interface - efficiency: your computer has limited memory, which graphical user interfaces use a lot of—the command line uses less @@ -71,7 +71,7 @@ The flags or options, such as `-n` in the example above, typically begin with a Spaces take on a special role when using the command line. For this reason, it's good practice to avoid spaces in file names. If you need to refer to a filename with spaces in, you’ll need to use quotes or escape the spaces in the file names using a `\`, for example `this is my file.txt` becomes `this\ is\ my\ file.txt` ::: -To run programmes from the command line, all you need is the name of the programme as the command: in fact, commands *are* programmes. The `date` command refers to an actual programme on your computer that you can find. And this also explains a bit of what's going on when you *run a script from the command line* (more on that later). +To run programmes from the command line, all you need is the name of the programme as the command: in fact, commands _are_ programmes. The `date` command refers to an actual programme on your computer that you can find. And this also explains a bit of what's going on when you _run a script from the command line_ (more on that later). Once you've run a few commands, you'll notice that you can't navigate around the command line like you can a text file or Python script. Here are some tips for navigating the command line: @@ -93,11 +93,11 @@ Once you've run a few commands, you'll notice that you can't navigate around the ### Navigating directories -While we're on navigating, it's useful to understand *where* in the computer you are when you open the command line. If you open a terminal pane within VS Code, you will start (by default at least) within the same folder as your project. Starting a terminal instance outside of VS Code will get you a terminal in a root directory for your computer; for example, on a Mac, opening a new terminal window starts you in `/Users/yourusername/`. +While we're on navigating, it's useful to understand _where_ in the computer you are when you open the command line. If you open a terminal pane within VS Code, you will start (by default at least) within the same folder as your project. Starting a terminal instance outside of VS Code will get you a terminal in a root directory for your computer; for example, on a Mac, opening a new terminal window starts you in `/Users/yourusername/`. To find out "where" you are when you open a terminal, you can use the `pwd` command, which stands for "print working directory". -The table below shows some useful commands for moving around your computer using the command line. Note that `cd` accepts a location *relative* to your current directory. +The table below shows some useful commands for moving around your computer using the command line. Note that `cd` accepts a location _relative_ to your current directory. | Command | What it does | | --------------------- | ------------------------------------------------------------ | @@ -124,7 +124,7 @@ Say you have a script called `analysis.py`, you can run it with Python on the co uv run python analysis.py ``` -which calls Python as a programme and gives it `analysis.py` as the argument. If you have multiple versions of Python, which you should do if you're following best practice and using a version per project, then you can see *which* version of Python is being used with +which calls Python as a programme and gives it `analysis.py` as the argument. If you have multiple versions of Python, which you should do if you're following best practice and using a version per project, then you can see _which_ version of Python is being used with ```bash which python @@ -134,39 +134,39 @@ which python Now we'll see some useful commands for the terminal. - | Command                                          | What it does | - | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | - | `man ` | Shows a manual for the given command | - | `touch ` | Creates an empty file named `` | - | `code ` | Open a file in VS Code (creating it, if it does not exist) | - | `mkdir ` | creates a new folder called `foldername` | - | `echo ` | Prints `` | - | `cat ` | Print the full contents of `` | - | `head ` | Print the start of a file | - | `tail ` | Print the end of a file | - | `> ` | Redirects output from screen to ``. For example, `echo "Hello World" > hello.txt` | - | `>> ` | Redirects output from screen to the end of ``, ie appends output rather than overwrites it | - | ` | ` | The pipe symbol: uses output from one command as input into another. For example, `head -n 10 data.csv | > hello_world.txt` would write the first 10 lines of data.csv into a file called hello_world.txt | - | `less ` | Print out the contents of a file in paginated form. Use `ctrl+v` and `Alt+v` (or `⌘+v` and `⌥+v` on Mac) to move up and down. Press `q` to quit. | - | `wc -l` | Returns number of lines in input, for example `cat | wc -l`. Use `wc` alone for word count. | - | `sort` | Arrange lines in a file in alphabetical order | - | `uniq` | Remove duplicate lines from input, for example `cat | uniq` or `uniq -d` to show duplicate files | - | `mv` | Move or rename a file; for example, `mv file1 file2` would rename `file1` to `file2` while `mv file1 ~` would move `file1` to the home directory | - | `cp` | Copy a file; for example, `cp file1 file2` would copy `file1` to `file2` while `cp file1 ~` would make a copy of `file1` in the home directory | - | `rm ` | Permanently remove a file | - | `rmdir ` | Permanently remove an empty directory | - | `rm -rf ` | ⚠ Permanently remove everything in a directory ⚠ | - | `grep ` | Search for a given term, for example `cat hello_world.txt | grep world` | - | `ls` | Basically, this means list stuff (files and folders) in the current directory | - | `ls -a` | List stuff in the current directory even if it's hidden | - | `ls -l` | List stuff in a more readable format and show permissions | - | `ls -S` | List stuff by size | - | `file ` | Give information on the file type of `` | - | `find` | Find specific files on your computer, can be piped into other commands for example `find *.md -size +5k -type f | xargs wc -l` will count the number of lines `wc -l` of all files, `-type f`, ending in `.md` that are greater than 5 kilobytes in size, `-size +5k`. | - | `diff -u ` | Show a single summary of the differences between two files. | + | Command                                          | What it does | + | --------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | + | `man ` | Shows a manual for the given command | + | `touch ` | Creates an empty file named `` | + | `code ` | Open a file in VS Code (creating it, if it does not exist) | + | `mkdir ` | creates a new folder called `foldername` | + | `echo ` | Prints `` | + | `cat ` | Print the full contents of `` | + | `head ` | Print the start of a file | + | `tail ` | Print the end of a file | + | `> ` | Redirects output from screen to ``. For example, `echo "Hello World" > hello.txt` | + | `>> ` | Redirects output from screen to the end of ``, ie appends output rather than overwrites it | + | ` | ` | The pipe symbol: uses output from one command as input into another. For example, `head -n 10 data.csv | > hello_world.txt` would write the first 10 lines of data.csv into a file called hello_world.txt | + | `less ` | Print out the contents of a file in paginated form. Use `ctrl+v` and `Alt+v` (or `⌘+v` and `⌥+v` on Mac) to move up and down. Press `q` to quit. | + | `wc -l` | Returns number of lines in input, for example `cat | wc -l`. Use `wc` alone for word count. | + | `sort` | Arrange lines in a file in alphabetical order | + | `uniq` | Remove duplicate lines from input, for example `cat | uniq`or`uniq -d` to show duplicate files | + | `mv` | Move or rename a file; for example, `mv file1 file2` would rename `file1` to `file2` while `mv file1 ~` would move `file1` to the home directory | + | `cp` | Copy a file; for example, `cp file1 file2` would copy `file1` to `file2` while `cp file1 ~` would make a copy of `file1` in the home directory | + | `rm ` | Permanently remove a file | + | `rmdir ` | Permanently remove an empty directory | + | `rm -rf ` | ⚠ Permanently remove everything in a directory ⚠ | + | `grep ` | Search for a given term, for example `cat hello_world.txt | grep world` | + | `ls` | Basically, this means list stuff (files and folders) in the current directory | + | `ls -a` | List stuff in the current directory even if it's hidden | + | `ls -l` | List stuff in a more readable format and show permissions | + | `ls -S` | List stuff by size | + | `file ` | Give information on the file type of `` | + | `find` | Find specific files on your computer, can be piped into other commands for example `find \*.md -size +5k -type f | xargs wc -l`will count the number of lines`wc -l`of all files,`-type f`, ending in `.md`that are greater than 5 kilobytes in size,`-size +5k`. | + | `diff -u ` | Show a single summary of the differences between two files. | ![More details of the grep command](https://pbs.twimg.com/media/DcPeD_CW0AEkSar?format=jpg&name=small) -*More details of the grep command, by [\@b0rk](https://twitter.com/b0rk).* +_More details of the grep command, by [\@b0rk](https://twitter.com/b0rk)._ You can write for loops in bash (remember, it's a language). The general structure is @@ -187,7 +187,7 @@ A more interesting example is giving the number of lines of text, number of word ```bash for i in $(ls *.csv) -do +do wc $i done ``` @@ -204,7 +204,7 @@ done A couple of new features appeared in the examples above. -`*` is a *wildcard character*, it tells bash to look for anything that ends in ".csv". This is not the only special case; `?` serves a similar purpose of standing in for any character but just *one* character rather than arbitrarily many. If you had a folder with `file1.csv`, `file2.csv`, etc., up to 9, then you could use `file?.csv` to refer to all of them but this would not pick up `file10.csv`. +`*` is a _wildcard character_, it tells bash to look for anything that ends in ".csv". This is not the only special case; `?` serves a similar purpose of standing in for any character but just _one_ character rather than arbitrarily many. If you had a folder with `file1.csv`, `file2.csv`, etc., up to 9, then you could use `file?.csv` to refer to all of them but this would not pick up `file10.csv`. Another special character we've already seen is the curly brace, `{}`. Whenever you have a common substring in a series of commands using curly braces tells the command line to expand what's in them automatically. In an example above, this is used on 1 to 5. But it can also be used in, for example, file names: @@ -272,7 +272,7 @@ You can find more of these special variables [here](https://tldp.org/LDP/abs/htm [**pandoc**](https://pandoc.org/) is absolutely brilliant: if you need to convert files containing text from one format to another, it really is a swiss-army knife. There isn't space here to list the ridiculous number of documents it can convert between, but, importantly, it can translate back and forth between all of the following: markdown, $\LaTeX$, Microsoft Word's docx, OpenOffice's ODT, HTML, and Jupyter Notebook. -It can also write from any of those formats (and more) in one direction *to* PDF, Microsoft Powerpoint, and $\LaTeX$ Beamer. +It can also write from any of those formats (and more) in one direction _to_ PDF, Microsoft Powerpoint, and $\LaTeX$ Beamer. To use **pandoc**, install it following the instructions on the website and then call it like this: @@ -284,9 +284,9 @@ This is an example where the input is a .tex document and the output, `-o`, is a You can get quite fancy with **pandoc**, for example you can translate a whole book's worth of latex into a Word doc complete with a Word style, a bibliography via biblatex, equations, and figures. Nothing can save Word from being painful to use, but **pandoc** certainly helps. -[**eza**](https://eza.rocks/) is an upgrade on the `ls` command. It is designed to be an improved file lister with more features and better defaults. It uses colours to distinguish file types and metadata. Follow the instructions on the website to install it on your operating system. To replace `ls` with `eza`, you can use a terminal *alias*. There's a good guide [available here](https://denisrasulev.medium.com/eza-the-best-ls-command-replacement-9621252323e). +[**eza**](https://eza.rocks/) is an upgrade on the `ls` command. It is designed to be an improved file lister with more features and better defaults. It uses colours to distinguish file types and metadata. Follow the instructions on the website to install it on your operating system. To replace `ls` with `eza`, you can use a terminal _alias_. There's a good guide [available here](https://denisrasulev.medium.com/eza-the-best-ls-command-replacement-9621252323e). -**nano** is a built-in text editor that runs *within* the terminal. This can be really useful if you're working on the cloud (but it's not got the rich features of a GUI-based text editor like VS Code). To open a file using **nano**, the command is `nano file.txt`. Nano displays instructions on how to navigate when it loads up but exiting is the hardest part: when you're done, hit `Ctrl+X`, then `y` to save, and then `enter` to exit. +**nano** is a built-in text editor that runs _within_ the terminal. This can be really useful if you're working on the cloud (but it's not got the rich features of a GUI-based text editor like VS Code). To open a file using **nano**, the command is `nano file.txt`. Nano displays instructions on how to navigate when it loads up but exiting is the hardest part: when you're done, hit `Ctrl+X`, then `y` to save, and then `enter` to exit. [**wget**](https://www.gnu.org/software/wget/) is a command-line utility for downloading files from the internet. It's very simple to use, the syntax is just `wget [options] [url]`. For example, to download the starwars csv file used in this book, the command is diff --git a/communicate-plots.ipynb b/communicate-plots.ipynb index 8531e79..0f0f567 100644 --- a/communicate-plots.ipynb +++ b/communicate-plots.ipynb @@ -11,28 +11,24 @@ "\n", "In this chapter, you'll learn about using visualisation to communicate.\n", "\n", - "In @sec-exploratory-data-analysis, you learned how to use plots as tools for *exploration*.\n", + "In @sec-exploratory-data-analysis, you learned how to use plots as tools for _exploration_.\n", "When you make exploratory plots, you know—even before looking—which variables the plot will display.\n", "You made each plot for a purpose, quickly looked at it, and then moved on to the next plot.\n", "In the course of most analyses, you'll produce tens or hundreds of plots, most of which are immediately thrown away.\n", "\n", - "Now that you understand your data, you need to *communicate* your understanding to others.\n", + "Now that you understand your data, you need to _communicate_ your understanding to others.\n", "Your audience will likely not share your background knowledge and will not be deeply invested in the data. To help others quickly build up a good mental model of the data, you will need to invest considerable effort in making your plots as self-explanatory as possible. In this chapter, you'll learn some of the tools that **lets-plot** provides to do make charts tell a story.\n", "\n", "### Prerequisities\n", "\n", - "As ever, there are a plethora of options (and packages) for data visualisation using code. We're focusing on the declarative, \"grammar of graphics\" approach using **lets-plot** here, but advanced users looking for more complex graphics might wish to use an imperative library such as the excellent **matplotlib**. You should have both **lets-plot** and **pandas** installed. Once you have them installed, import them like so:" + "As ever, there are a plethora of options (and packages) for data visualisation using code. We're focusing on the declarative, \"grammar of graphics\" approach using **lets-plot** here, but advanced users looking for more complex graphics might wish to use an imperative library such as **Plotly Graph Objects**. You should have both **lets-plot** and **polars** installed. Once you have them installed, import them like so:\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "51a55374", - "metadata": { - "tags": [ - "remove-cell" - ] - }, + "id": "78eeea41", + "metadata": {}, "outputs": [], "source": [ "# remove cell\n", @@ -52,7 +48,7 @@ "outputs": [], "source": [ "import numpy as np\n", - "import pandas as pd\n", + "import polars as pl\n", "from lets_plot import *\n", "\n", "LetsPlot.setup_html()" @@ -65,7 +61,7 @@ "source": [ "## Labels, titles, and other contextual information\n", "\n", - "The easiest place to start when turning an exploratory graphic into an expository graphic is with good labels. Let's look at an example using the MPG (miles per gallon) data, which covers the fuel economy for 38 popular models of cars from 1999 to 2008." + "The easiest place to start when turning an exploratory graphic into an expository graphic is with good labels. Let's look at an example using the MPG (miles per gallon) data, which covers the fuel economy for 38 popular models of cars from 1999 to 2008.\n" ] }, { @@ -76,8 +72,9 @@ "outputs": [], "source": [ "# load the data\n", - "mpg = pd.read_csv(\n", - " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/mpg.csv\", index_col=0\n", + "mpg = pl.read_csv(\n", + " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/mpg.csv\",\n", + " row_index_offset=0,\n", ")" ] }, @@ -86,7 +83,7 @@ "id": "1813ab08", "metadata": {}, "source": [ - "We want to show fuel efficiency on the highway changes with engine displacement, in litres. The most basic chart we can do with these variables is:" + "We want to show fuel efficiency on the highway changes with engine displacement, in litres. The most basic chart we can do with these variables is:\n" ] }, { @@ -114,7 +111,7 @@ "- remove the y-axis label that is at an awkward viewing angle\n", "- add a caption with the source of the data\n", "\n", - "Putting this all in, we get:" + "Putting this all in, we get:\n" ] }, { @@ -143,9 +140,9 @@ "id": "2e28877a", "metadata": {}, "source": [ - "This is much clearer. It's easier to read, we know where the data come from, and we can see *why* we're being shown it too.\n", + "This is much clearer. It's easier to read, we know where the data come from, and we can see _why_ we're being shown it too.\n", "\n", - "But maybe we want a different message? You can flex depending on your needs, and some people prefer to have a rotated y-axis so that the subtitle can provide even more context:" + "But maybe we want a different message? You can flex depending on your needs, and some people prefer to have a rotated y-axis so that the subtitle can provide even more context:\n" ] }, { @@ -180,7 +177,7 @@ "1. Create one plot on the fuel economy data with customized `title`, `subtitle`, `caption`, `x`, `y`, and `color` labels.\n", "\n", "2. Recreate the following plot using the fuel economy data.\n", - " Note that both the colours and shapes of points vary by type of drive train." + " Note that both the colours and shapes of points vary by type of drive train.\n" ] }, { @@ -211,7 +208,7 @@ "id": "e10cdbd9", "metadata": {}, "source": [ - "3. Take an exploratory graphic that you've created in the last month, and add informative titles to make it easier for others to understand." + "3. Take an exploratory graphic that you've created in the last month, and add informative titles to make it easier for others to understand.\n" ] }, { @@ -229,7 +226,7 @@ "There are two possible sources of labels: ones that are part of the data, which we'll add with `geom_text()`; and ones that we add directly and manually as annotations using `geom_label()`.\n", "\n", "In the first case, you might have a data frame that contains labels.\n", - "In the following plot we pull out the cars with the highest engine size in each drive type and save their information as a new data frame called `label_info`. In creating it, we pick out the mean values of \"hwy\" by \"drv\" as the points to label—but we could do any aggregation we feel would work well on the chart." + "In the following plot we pull out the cars with the highest engine size in each drive type and save their information as a new data frame called `label_info`. In creating it, we pick out the mean values of \"hwy\" by \"drv\" as the points to label—but we could do any aggregation we feel would work well on the chart.\n" ] }, { @@ -245,11 +242,13 @@ " \"r\": \"rear-wheel drive\",\n", "}\n", "label_info = (\n", - " mpg.groupby(\"drv\")\n", - " .agg({\"hwy\": \"mean\", \"displ\": \"mean\"})\n", - " .reset_index()\n", - " .assign(drive_type=lambda x: x[\"drv\"].map(mapping))\n", - " .round(2)\n", + " mpg.group_by(\"drv\")\n", + " .agg(pl.mean(\"hwy\").round(2), pl.mean(\"displ\").round(2))\n", + " .with_columns(\n", + " pl.col(\"drv\")\n", + " .replace_strict(mapping, return_dtype=pl.String)\n", + " .alias(\"drive_type\")\n", + " )\n", ")\n", "label_info" ] @@ -259,7 +258,7 @@ "id": "93a136fe", "metadata": {}, "source": [ - "Then, we use this new data frame to directly label the three groups to replace the legend with labels placed directly on the plot. Using the fontface and size arguments we can customize the look of the text labels. They’re larger than the rest of the text on the plot and bolded. (`theme(legend.position = \"none\")` turns all the legends off — we’ll talk about it more shortly.)" + "Then, we use this new data frame to directly label the three groups to replace the legend with labels placed directly on the plot. Using the fontface and size arguments we can customize the look of the text labels. They’re larger than the rest of the text on the plot and bolded. (`theme(legend.position = \"none\")` turns all the legends off — we’ll talk about it more shortly.)\n" ] }, { @@ -292,8 +291,7 @@ "source": [ "Note the use of `hjust` (horizontal justification) and `vjust` (vertical justification) to control the alignment of the label.\n", "\n", - "\n", - "The second of the two methods we're looking at is `geom_label()`. This has two modes: in the first, it works like `geom_text()` but with a box around the text, like so:" + "The second of the two methods we're looking at is `geom_label()`. This has two modes: in the first, it works like `geom_text()` but with a box around the text, like so:\n" ] }, { @@ -303,7 +301,9 @@ "metadata": {}, "outputs": [], "source": [ - "potential_outliers = mpg.query(\"hwy > 40 | (hwy > 20 & displ > 5)\")\n", + "potential_outliers = mpg.filter(\n", + " (pl.col(\"hwy\") > 40) | ((pl.col(\"hwy\") > 20) & (pl.col(\"displ\") > 5))\n", + ")\n", "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", " + geom_point(color=\"black\")\n", @@ -331,7 +331,7 @@ "id": "38b69dcf", "metadata": {}, "source": [ - "The second method is generally useful for adding either a single or several annotations to a plot, like so:" + "The second method is generally useful for adding either a single or several annotations to a plot, like so:\n" ] }, { @@ -368,7 +368,7 @@ "id": "0720e7eb", "metadata": {}, "source": [ - "Annotation is a powerful tool for communicating main takeaways and interesting features of your visualisations. The only limit is your imagination (and your patience with positioning annotations to be aesthetically pleasing)!" + "Annotation is a powerful tool for communicating main takeaways and interesting features of your visualisations. The only limit is your imagination (and your patience with positioning annotations to be aesthetically pleasing)!\n" ] }, { @@ -379,15 +379,15 @@ "Remember, in addition to `geom_text()` and `geom_label()`, you have many other geoms in **lets-plot** available to help annotate your plot.\n", "A couple ideas:\n", "\n", - "- Use `geom_hline()` and `geom_vline()` to add reference lines.\n", - " We often make them thick (`size = 2`) and grey (`color = gray`), and draw them underneath the primary data layer.\n", - " That makes them easy to see, without drawing attention away from the data.\n", + "- Use `geom_hline()` and `geom_vline()` to add reference lines.\n", + " We often make them thick (`size = 2`) and grey (`color = gray`), and draw them underneath the primary data layer.\n", + " That makes them easy to see, without drawing attention away from the data.\n", "\n", - "- Use `geom_rect()` to draw a rectangle around points of interest.\n", - " The boundaries of the rectangle are defined by aesthetics `xmin`, `xmax`, `ymin`, `ymax`.\n", + "- Use `geom_rect()` to draw a rectangle around points of interest.\n", + " The boundaries of the rectangle are defined by aesthetics `xmin`, `xmax`, `ymin`, `ymax`.\n", "\n", - "- You already saw the use of `geom_segment()` with the `arrow` argument to draw attention to a point with an arrow.\n", - " Use aesthetics `x` and `y` to define the starting location, and `xend` and `yend` to define the end location.\n" + "- You already saw the use of `geom_segment()` with the `arrow` argument to draw attention to a point with an arrow.\n", + " Use aesthetics `x` and `y` to define the starting location, and `xend` and `yend` to define the end location.\n" ] }, { @@ -445,7 +445,7 @@ " scale_y_continuous() +\n", " scale_color_discrete()\n", ")\n", - "```" + "```\n" ] }, { @@ -461,11 +461,11 @@ "The default scales have been carefully chosen to do a good job for a wide range of inputs.\n", "Nevertheless, you might want to override the defaults for two reasons:\n", "\n", - "- You might want to tweak some of the parameters of the default scale.\n", - " This allows you to do things like change the breaks on the axes, or the key labels on the legend.\n", + "- You might want to tweak some of the parameters of the default scale.\n", + " This allows you to do things like change the breaks on the axes, or the key labels on the legend.\n", "\n", - "- You might want to replace the scale altogether, and use a completely different algorithm.\n", - " Often you can do better than the default because you know more about the data.\n" + "- You might want to replace the scale altogether, and use a completely different algorithm.\n", + " Often you can do better than the default because you know more about the data.\n" ] }, { @@ -478,8 +478,8 @@ "Collectively axes and legends get the somewhat confusing name **guides** in **lets-plot**. Axes are used for x and y aesthetics; legends are used for everything else.\n", "\n", "There are two primary arguments that affect the appearance of the ticks on the axes and the keys on the legend: `breaks` and `labels`.\n", - "Breaks controls the position of the ticks, or the values associated with the keys. If you like, the breaks *are* the ticks.\n", - "Labels controls the text label associated with each tick/key. We might more accurately call these *tick labels*.\n", + "Breaks controls the position of the ticks, or the values associated with the keys. If you like, the breaks _are_ the ticks.\n", + "Labels controls the text label associated with each tick/key. We might more accurately call these _tick labels_.\n", "The most common use of `breaks` is to override the default choice:\n" ] }, @@ -534,7 +534,7 @@ "- the .2 says use two significant digits\n", "- the s says, use the Système International (SI)\n", "\n", - "There are a wealth of alternative options for formatting—it's best to use the [helpful page on formatting](https://lets-plot.org/pages/formats.html) in the documentation of **lets-plot** to find out more." + "There are a wealth of alternative options for formatting—it's best to use the [helpful page on formatting](https://lets-plot.org/pages/formats.html) in the documentation of **lets-plot** to find out more.\n" ] }, { @@ -544,17 +544,16 @@ "metadata": {}, "outputs": [], "source": [ - "diamonds = pd.read_csv(\n", + "diamonds = pl.read_csv(\n", " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/diamonds.csv\",\n", - " index_col=0,\n", + " row_index_offset=0,\n", ")\n", - "diamonds[\"cut\"] = diamonds[\"cut\"].astype(\n", - " pd.CategoricalDtype(\n", - " categories=[\"Fair\", \"Good\", \"Very Good\", \"Premium\", \"Ideal\"], ordered=True\n", - " )\n", + "\n", + "diamonds = diamonds.with_columns(\n", + " pl.col(\"cut\").cast(pl.Enum([\"Fair\", \"Good\", \"Very Good\", \"Premium\", \"Ideal\"]))\n", ")\n", - "diamonds[\"color\"] = diamonds[\"color\"].astype(\n", - " pd.CategoricalDtype(categories=[\"D\", \"E\", \"F\", \"G\", \"H\", \"I\", \"J\"], ordered=True)\n", + "diamonds = diamonds.with_columns(\n", + " pl.col(\"color\").cast(pl.Enum([\"D\", \"E\", \"F\", \"G\", \"H\", \"I\", \"J\"]))\n", ")" ] }, @@ -578,7 +577,7 @@ "id": "6f2edc1b", "metadata": {}, "source": [ - "Another use of breaks is when you have relatively few data points and want to highlight exactly where the observations occur. For example, take this plot that shows when each US president started and ended their term." + "Another use of breaks is when you have relatively few data points and want to highlight exactly where the observations occur. For example, take this plot that shows when each US president started and ended their term.\n" ] }, { @@ -588,19 +587,22 @@ "metadata": {}, "outputs": [], "source": [ - "presidential = pd.read_csv(\n", + "presidential = pl.read_csv(\n", " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/presidential.csv\",\n", - " index_col=0,\n", + " row_index_offset=0,\n", + ")\n", + "presidential = presidential.with_columns(\n", + " pl.col(\"start\").str.strptime(pl.Date, \"%Y-%m-%d\").dt.year(),\n", + " pl.col(\"end\").str.strptime(pl.Date, \"%Y-%m-%d\").dt.year(),\n", + " (pl.col(\"rownames\") + 33).alias(\"id\"),\n", ")\n", - "presidential = presidential.astype({\"start\": \"datetime64[ns]\", \"end\": \"datetime64[ns]\"})\n", - "presidential[\"id\"] = 33 + presidential.index\n", "presidential.head()" ] }, { "cell_type": "code", "execution_count": null, - "id": "cd2cc430", + "id": "7d88976d", "metadata": {}, "outputs": [], "source": [ @@ -624,7 +626,7 @@ "\n", "To control the overall position of the legend, you need to use a `theme()` setting.\n", "We'll come back to themes at the end of the chapter, but in brief, they control the non-data parts of the plot.\n", - "The theme setting `legend.position` controls where the legend is drawn, and to demonstrate this we'll use `gggrid()` to arrange all of the plots." + "The theme setting `legend.position` controls where the legend is drawn, and to demonstrate this we'll use `gggrid()` to arrange all of the plots.\n" ] }, { @@ -651,7 +653,7 @@ "source": [ "If your plot is short and wide, place the legend at the top or bottom, and if it's tall and narrow, place the legend at the left or right. You can also use `legend_position = \"none\"` to suppress the display of the legend altogether.\n", "\n", - "To control the display of individual legends, use `guides()` along with `guide_legend()` or `guide_colorbar()`." + "To control the display of individual legends, use `guides()` along with `guide_legend()` or `guide_colorbar()`.\n" ] }, { @@ -659,7 +661,6 @@ "id": "f27913c7", "metadata": {}, "source": [ - "\n", "### Replacing a scale\n", "\n", "Instead of just tweaking the details a little, you can instead replace the scale altogether.\n", @@ -667,7 +668,7 @@ "Fortunately, the same principles apply to all the other aesthetics, so once you've mastered position and colour, you'll be able to quickly pick up other scale replacements.\n", "\n", "It's very useful to plot transformations of your variable.\n", - "For example, it's easier to see the precise relationship between `carat` and `price` if we log transform them. The way to do this is by using an `apply()` function on the data that gets sent to `ggplot`:" + "For example, it's easier to see the precise relationship between `carat` and `price` if we log transform them. The way to do this is by using an `apply()` function on the data that gets sent to `ggplot`:\n" ] }, { @@ -679,7 +680,7 @@ "source": [ "(\n", " ggplot(\n", - " diamonds.apply({\"carat\": np.log10, \"price\": np.log10}),\n", + " diamonds.with_columns(pl.col(\"carat\").log10(), pl.col(\"price\").log10()),\n", " aes(x=\"carat\", y=\"price\"),\n", " )\n", " + geom_bin2d()\n", @@ -691,7 +692,7 @@ "id": "f19dbbec", "metadata": {}, "source": [ - "However, the disadvantage of this transformation is that the axes are now mislabelled with the original values, making it hard to interpret the plot. Instead of doing the transformation in the aesthetic mapping, we can instead do it with the scale. This is visually identical, except the axes are labelled on the original data scale." + "However, the disadvantage of this transformation is that the axes are now mislabelled with the original values, making it hard to interpret the plot. Instead of doing the transformation in the aesthetic mapping, we can instead do it with the scale. This is visually identical, except the axes are labelled on the original data scale.\n" ] }, { @@ -714,7 +715,7 @@ "id": "4402c4de", "metadata": {}, "source": [ - "Another scale that is frequently customised is colour. The default categorical scale picks colors that are evenly spaced around the color wheel. Useful alternatives are the ColorBrewer scales which have been hand tuned to work better for people with common types of colour blindness. The two plots below look similar, but there is enough difference in the shades of red and green that the dots on the right can be distinguished even by people with red-green colour blindness." + "Another scale that is frequently customised is colour. The default categorical scale picks colors that are evenly spaced around the color wheel. Useful alternatives are the ColorBrewer scales which have been hand tuned to work better for people with common types of colour blindness. The two plots below look similar, but there is enough difference in the shades of red and green that the dots on the right can be distinguished even by people with red-green colour blindness.\n" ] }, { @@ -748,7 +749,7 @@ "source": [ "Don't forget simpler techniques for improving accessibility.\n", "If there are just a few colors, you can add a redundant shape mapping.\n", - "This will also help ensure your plot is interpretable in black and white." + "This will also help ensure your plot is interpretable in black and white.\n" ] }, { @@ -756,7 +757,7 @@ "id": "253af5a4", "metadata": {}, "source": [ - "The ColorBrewer scales are documented online at . The sequential (top) and diverging (bottom) palettes are particularly useful if your categorical values are ordered, or have a \"middle\". This often arises if you've used `pd.cut()` to make a continuous variable into a categorical variable." + "The ColorBrewer scales are documented online at . The sequential (top) and diverging (bottom) palettes are particularly useful if your categorical values are ordered, or have a \"middle\". This often arises if you've used `pd.cut()` to make a continuous variable into a categorical variable.\n" ] }, { @@ -905,7 +906,7 @@ "id": "c32c2237", "metadata": {}, "source": [ - "When you have a predefined mapping between values and colours, use `scale_color_manual()`. For example, if we map presidential party to colour, we want to use the standard mapping of red for Republicans and blue for Democrats. One approach for assigning these colors is using hex colour codes:" + "When you have a predefined mapping between values and colours, use `scale_color_manual()`. For example, if we map presidential party to colour, we want to use the standard mapping of red for Republicans and blue for Democrats. One approach for assigning these colors is using hex colour codes:\n" ] }, { @@ -915,13 +916,13 @@ "metadata": {}, "outputs": [], "source": [ - "mini_presid = presidential.iloc[5:, :]\n", + "mini_presid = presidential.slice(5)\n", "\n", "(\n", " ggplot(mini_presid, aes(x=\"start\", y=\"id\", color=\"party\"))\n", " + geom_point(size=3)\n", " + geom_segment(aes(xend=\"end\", yend=\"id\"), size=1)\n", - " + scale_x_datetime(breaks=mini_presid[\"start\"], format=\"%Y\")\n", + " + scale_x_datetime(breaks=mini_presid[\"start\"].to_list(), format=\"%Y\")\n", " + scale_color_manual(values=[\"#00AEF3\", \"#E81B23\"], name=\"party\")\n", ")" ] @@ -936,7 +937,7 @@ "For continuous colour, you can use the built-in `scale_color_gradient()` or `scale_fill_gradient()`.\n", "If you have a diverging scale, you can use `scale_color_gradient2()`. That allows you to give, for example, positive and negative values different colors. That's sometimes also useful if you want to distinguish points above or below the mean.\n", "\n", - "Another option is to use the viridis, magma, inferno, and plasma color scales developed for the extremely powerful imperative Python plotting package **[matplotlib](https://matplotlib.org/)**. The designers, Nathaniel Smith and Stéfan van der Walt, carefully tailored continuous color schemes that are perceptible to people with various forms of color blindness as well as perceptually uniform in both color and black and white. These scales are available as palettes in *lets-plot*. Here's an example using the continuous version of viridis (we'll generate some random data first):" + "Another option is to use the viridis, magma, inferno, and plasma color scales developed for the extremely powerful imperative Python plotting package **[matplotlib](https://matplotlib.org/)**. The designers, Nathaniel Smith and Stéfan van der Walt, carefully tailored continuous color schemes that are perceptible to people with various forms of color blindness as well as perceptually uniform in both color and black and white. These scales are available as palettes in _lets-plot_. Here's an example using the continuous version of viridis (we'll generate some random data first):\n" ] }, { @@ -947,7 +948,7 @@ "outputs": [], "source": [ "prng = np.random.default_rng(1837) # prng=probabilistic random number generator\n", - "df_rnd = pd.DataFrame(prng.standard_normal((1000, 2)), columns=[\"x\", \"y\"])\n", + "df_rnd = pl.DataFrame(prng.standard_normal((1000, 2)), schema=[\"x\", \"y\"])\n", "(\n", " ggplot(df_rnd, aes(x=\"x\", y=\"y\"))\n", " + geom_bin2d()\n", @@ -1002,7 +1003,7 @@ ")\n", "\n", "(\n", - " ggplot(mpg.loc[mpg_condition], aes(x=\"displ\", y=\"hwy\"))\n", + " ggplot(mpg.filter(mpg_condition), aes(x=\"displ\", y=\"hwy\"))\n", " + geom_point(aes(color=\"drv\"))\n", " + geom_smooth(method=\"loess\")\n", ")" @@ -1015,7 +1016,7 @@ "source": [ "Let's compare these to the two plots below where the first plot sets the `limits` on individual scales and the second plot sets them in `coord_cartesian()`.\n", "We can see that reducing the limits is equivalent to subsetting the data.\n", - "Therefore, to zoom in on a region of the plot, it's generally best to use `coord_cartesian()`." + "Therefore, to zoom in on a region of the plot, it's generally best to use `coord_cartesian()`.\n" ] }, { @@ -1054,8 +1055,8 @@ "id": "5d1fc3ee", "metadata": {}, "source": [ - "On the other hand, setting the `limits` on individual scales is generally more useful if you want to *expand* the limits, e.g., to match scales across different plots.\n", - "For example, if we extract two classes of cars and plot them separately, it's difficult to compare the plots because all three scales (the x-axis, the y-axis, and the colour aesthetic) have different ranges." + "On the other hand, setting the `limits` on individual scales is generally more useful if you want to _expand_ the limits, e.g., to match scales across different plots.\n", + "For example, if we extract two classes of cars and plot them separately, it's difficult to compare the plots because all three scales (the x-axis, the y-axis, and the colour aesthetic) have different ranges.\n" ] }, { @@ -1065,8 +1066,8 @@ "metadata": {}, "outputs": [], "source": [ - "suv = mpg.loc[mpg[\"class\"] == \"suv\"]\n", - "compact = mpg.loc[mpg[\"class\"] == \"compact\"]\n", + "suv = mpg.filter(mpg[\"class\"] == \"suv\")\n", + "compact = mpg.filter(mpg[\"class\"] == \"compact\")\n", "(ggplot(suv, aes(x=\"displ\", y=\"hwy\", color=\"drv\")) + geom_point())" ] }, @@ -1095,9 +1096,9 @@ "metadata": {}, "outputs": [], "source": [ - "x_scale = scale_x_continuous(limits=mpg[\"displ\"].agg([\"max\", \"min\"]).tolist())\n", - "y_scale = scale_y_continuous(limits=mpg[\"hwy\"].agg([\"max\", \"min\"]).tolist())\n", - "col_scale = scale_color_discrete(limits=mpg[\"drv\"].unique())" + "x_scale = scale_x_continuous(limits=[mpg[\"displ\"].min(), mpg[\"displ\"].max()])\n", + "y_scale = scale_y_continuous(limits=[mpg[\"hwy\"].min(), mpg[\"hwy\"].max()])\n", + "col_scale = scale_color_discrete(limits=mpg[\"drv\"].unique().to_list)" ] }, { @@ -1152,11 +1153,11 @@ "\n", "2. Change the display of the presidential terms by:\n", "\n", - " a. Combining the two variants that customize colors and x axis breaks.\n", - " b. Improving the display of the y axis.\n", - " c. Labelling each term with the name of the president.\n", - " d. Adding informative plot labels.\n", - " e. Placing breaks every 4 years (this is trickier than it seems!).\n" + " a. Combining the two variants that customize colors and x axis breaks.\n", + " b. Improving the display of the y axis.\n", + " c. Labelling each term with the name of the president.\n", + " d. Adding informative plot labels.\n", + " e. Placing breaks every 4 years (this is trickier than it seems!).\n" ] }, { @@ -1191,7 +1192,7 @@ "source": [ "**lets-plot** includes several built-in themes that you can find [here](https://lets-plot.org/pages/api.html#predefined-themes). You can also create your own themes, if you are trying to match a particular corporate or journal style.\n", "\n", - "Here's an example of changing multiple `theme()` settings:" + "Here's an example of changing multiple `theme()` settings:\n" ] }, { @@ -1257,9 +1258,9 @@ "source": [ "## Saving plots to file\n", "\n", - "There are lots of output options to choose from to save your file to. Remember that, for graphics, *vector formats* are generally better than *raster formats*. In practice, this means saving plots in svg or pdf formats over jpg or png file formats. The svg format works in a lot of contexts (including Microsoft Word) and is a good default. To choose between formats, just supply the file extension and the file type will change automatically, eg \"chart.svg\" for svg or \"chart.png\" for png (thought note that raster formats often have extra options, like how many dots per inch to use).\n", + "There are lots of output options to choose from to save your file to. Remember that, for graphics, _vector formats_ are generally better than _raster formats_. In practice, this means saving plots in svg or pdf formats over jpg or png file formats. The svg format works in a lot of contexts (including Microsoft Word) and is a good default. To choose between formats, just supply the file extension and the file type will change automatically, eg \"chart.svg\" for svg or \"chart.png\" for png (thought note that raster formats often have extra options, like how many dots per inch to use).\n", "\n", - "Let's try this out using the figure we made in the previous exercise, `p1`. `path=\".\"` just drops the file in the current directory." + "Let's try this out using the figure we made in the previous exercise, `p1`. `path=\".\"` just drops the file in the current directory.\n" ] }, { @@ -1277,7 +1278,7 @@ "id": "7781794a", "metadata": {}, "source": [ - "To double check this has worked, let's use the terminal. We'll try the command `ls`, which lists everything in directory, and `grep *.svg` to pull out any files that end in `.svg` from what is returned by `ls`. These are strung together as commands by a `|`. (Note that the leading exclamation mark below just tells the software that builds this book to use the terminal.)" + "To double check this has worked, let's use the terminal. We'll try the command `ls`, which lists everything in directory, and `grep *.svg` to pull out any files that end in `.svg` from what is returned by `ls`. These are strung together as commands by a `|`. (Note that the leading exclamation mark below just tells the software that builds this book to use the terminal.)\n" ] }, { @@ -1319,7 +1320,7 @@ "\n", "While you've so far learned about how to make many different types of plots and how to customise them using a variety of techniques, we've barely scratched the surface of what you can create with **lets-plot**.\n", "\n", - "The best place to go for further information is the [**lets-plot** dcoumentation](https://lets-plot.org/)." + "The best place to go for further information is the [**lets-plot** dcoumentation](https://lets-plot.org/).\n" ] } ], @@ -1331,7 +1332,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "python4ds", "language": "python", "name": "python3" }, @@ -1345,7 +1346,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/data-import.ipynb b/data-import.ipynb index 59363eb..9852811 100644 --- a/data-import.ipynb +++ b/data-import.ipynb @@ -13,7 +13,7 @@ "\n", "### Prerequisites\n", "\n", - "You will need to have the **pandas** package installed. You'll need to ensure you have **pandas** installed. To do this, and to import **pandas** into your session, run" + "You will need to have the **pandas** package installed. You'll need to ensure you have **pandas** installed. To do this, and to import **pandas** into your session, run\n" ] }, { @@ -23,7 +23,7 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd" + "import polars as pl" ] }, { @@ -31,7 +31,7 @@ "id": "e29b7103", "metadata": {}, "source": [ - "If this command fails, you don't have **pandas** installed. Open up the terminal in Visual Studio Code (Terminal -> New Terminal), `cd` to the folder you are working in, and type in `uv add pandas`. Note that once **pandas** is installed, the convention is to import it into your Python session under the name `pd` by putting `import pandas as pd` at the top of your script." + "If this command fails, you don't have **polars** installed. Open up the terminal in Visual Studio Code (Terminal -> New Terminal), `cd` to the folder you are working in, and type in `uv add polars`. Note that once **polars** is installed, the convention is to import it into your Python session under the name `pl` by putting `import polars as pl` at the top of your script.\n" ] }, { @@ -43,15 +43,13 @@ "\n", "There are a huge range of input and output formats available in **pandas**: Stata (.dta), Excel (.xls, .xlsx), csv, tsv, big data formats (HDF5, parquet), JSON, SAS, SPSS, SQL, and more; there's a [full list](https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html) of formats available in the documentation.\n", "\n", - "![From the pandas documentation](https://pandas.pydata.org/pandas-docs/stable/_images/02_io_readwrite.svg)\n", - "\n", - "While **pandas** has a huge number of ways to read data in and load it into your Python session, here we'll focus on the humble plain-text table file; for example csv (comma separated values) and tsv (tab separated values).\n", + "While **polars** has a huge number of ways to read data in and load it into your Python session, here we'll focus on the humble plain-text table file; for example csv (comma separated values) and tsv (tab separated values).\n", "\n", "### Reading data from a file\n", "\n", - "All of the power needed to open plain-text table files is contained in a single function, `pd.read_csv()`. It takes numerous arguments but the two most important are the (unnamed) first one, which gives the path to the data, and `sep=` (a keyword argument) that tells **pandas** whether to expect values to be separated by commas or tabs or another character; however, if you leave this field blank, **pandas** will guess for you. To see the full set of arguments, run `help(pd.read_csv)`.\n", + "All of the power needed to open plain-text table files is contained in a single function, `pl.read_csv()`. It takes numerous arguments but the two most important are the (unnamed) first one, which gives the path to the data, and `seperator=` (a keyword argument) that tells **polars** whether to expect values to be separated by commas or tabs or another character; however, if you leave this field blank, **polars** will guess for you. To see the full set of arguments, run `help(pl.read_csv)`.\n", "\n", - "Here is what a simple CSV file with a row for column names (also commonly referred to as the header row) and six rows of data looks like (using the terminal):" + "Here is what a simple CSV file with a row for column names (also commonly referred to as the header row) and six rows of data looks like (using the terminal):\n" ] }, { @@ -69,7 +67,7 @@ "id": "c7352c95", "metadata": {}, "source": [ - "Note that this is a CSV file, so the values are separated by commas. Now let's load this into a **pandas** dataframe in Python:" + "Note that this is a CSV file, so the values are separated by commas. Now let's load this into a **pandas** dataframe in Python:\n" ] }, { @@ -79,7 +77,7 @@ "metadata": {}, "outputs": [], "source": [ - "students = pd.read_csv(\"data/students.csv\")\n", + "students = pl.read_csv(\"data/students.csv\")\n", "students" ] }, @@ -96,7 +94,7 @@ "os.getcwd() # get current working directory (cwd)\n", "```\n", "\n", - "Say this comes back with 'python4DS', then your downloaded data should be in 'python4DS/data/students.csv'." + "Say this comes back with 'python4DS', then your downloaded data should be in 'python4DS/data/students.csv'.\n" ] }, { @@ -104,17 +102,17 @@ "id": "2f3979fc", "metadata": {}, "source": [ - "The first argument to `read_csv()` was the path to the data, and **pandas** guessed that this file uses commas as the separator.\n", + "The first argument to `read_csv()` was the path to the data, and **polars** guessed that this file uses commas as the separator.\n", "\n", - "The read CSV function automatically creates a new *index* (which is just the position of each row) and takes the top line of data as the *header* or *column names*. But you may wish to tweak this behaviour, in several ways.\n", + "The read CSV function automatically creates a new _index_ (which is just the position of each row) and takes the top line of data as the _header_ or _column names_. But you may wish to tweak this behaviour, in several ways.\n", "\n", - "1. Sometimes there are a few lines of metadata at the top of the file. You can use `skiprows=n` to skip the first `n` lines, eg `pd.read_csv(\"data/students.csv\", skiprows=2)`.\n", + "1. Sometimes there are a few lines of metadata at the top of the file. You can use `skip_rows=n` to skip the first `n` lines, eg `pl.read_csv(\"data/students.csv\", skip_rows=2)`.\n", "\n", - "2. The data might not have column names. You can use `names = ` a list to tell `read_csv()` to use a different option for the column names. For example, `pd.read_csv(\"data/students.csv\", names=range(5))` would put the numbers 0 to 4 in as the column names.\n", + "2. The data might not have column names. You can use `columns= ` a list to tell `read_csv()` to use a different option for the column names. For example, `pl.read_csv(\"data/students.csv\", columns=range(5))` would put the numbers 0 to 4 in as the column names.\n", "\n", - "3. You may wish to change which column is used as the index. The default behaviour is to create an index, but for this data we see that there already is an ID column we could use. To do this, use the `index_col=` argument, for example `pd.read_csv(\"data/students.csv\", index_col=0)`.\n", + "3. You may wish to inset a column to be used as a row index. For this data we see that there already is an ID column we could use. To do this, use the `row_index_offset=` argument, for example `pd.read_csv(\"data/students.csv\", row_index_offset=0)`.\n", "\n", - "This is all you need to know to read \\~75% of CSV files that you'll encounter in practice. Reading tab separated files and fixed width files is done with the same function." + "This is all you need to know to read \\~75% of CSV files that you'll encounter in practice. Reading tab separated files and fixed width files is done with the same function.\n" ] }, { @@ -130,7 +128,7 @@ "\n", "You might consider renaming them one by one with `.rename()` or you might use a convenience function from another package to clean them and turn them all into snake case at once. We will make use of the **skimpy** package to do this. Install it by running `uv add skimpy` in the terminal.\n", "\n", - "From **skimpy**, we will use the `clean_columns()` function; this takes in a data frame and returns a data frame with variable names converted to snake case." + "From **skimpy**, we will use the `clean_columns()` function; this takes in a data frame and returns a data frame with variable names converted to snake case.\n" ] }, { @@ -151,26 +149,7 @@ "id": "69c8a260", "metadata": {}, "source": [ - "Another common task after reading in data is to consider variable types. In the `favourite_food` column, there are a bunch of food items and then the value `NaN`, which has been read in as a floating point number rather than a missing string. We can solve this by casting that column to explicitly be composed of strings:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b4dad370", - "metadata": {}, - "outputs": [], - "source": [ - "students[\"favourite_food\"] = students[\"favourite_food\"].astype(\"string\")\n", - "students" - ] - }, - { - "cell_type": "markdown", - "id": "e85148b3", - "metadata": {}, - "source": [ - "Similarly, `\"age\"` has mixed data types: string and integer! Let's map the 'five' into the number five." + "Another common task after reading in data is to consider variable types. In the `age` column, there are mixed data types: string and integer!, which has been read in as a string. We can solve this by mapping the 'five' into the number five, then casting that column to explicitly be composed of integers:\n" ] }, { @@ -180,7 +159,7 @@ "metadata": {}, "outputs": [], "source": [ - "students[\"age\"] = students[\"age\"].replace(\"five\", 5)\n", + "students = students.with_columns(pl.col(\"age\").replace(\"five\", 5).cast(pl.Int64))\n", "students[\"age\"]" ] }, @@ -189,7 +168,7 @@ "id": "719f90cc", "metadata": {}, "source": [ - "In a moment, we will turn this into a column of integers too." + "In a moment, we will turn this into a column of integers too.\n" ] }, { @@ -197,7 +176,7 @@ "id": "e5c2b01e", "metadata": {}, "source": [ - "Another example where the data type is wrong is `meal_type`. This is a categorical variable with a known set of possible values. **pandas** has a special data type for these:" + "Another example where the data type is wrong is `meal_plan`. This is a categorical variable with a known set of possible values. **polars** has a special data type for these:\n" ] }, { @@ -207,7 +186,7 @@ "metadata": {}, "outputs": [], "source": [ - "students[\"meal_plan\"] = students[\"meal_plan\"].astype(\"category\")\n", + "students = students.with_columns(pl.col(\"meal_plan\").cast(pl.Categorical))\n", "students[\"meal_plan\"]" ] }, @@ -216,9 +195,9 @@ "id": "1e2a32a8", "metadata": {}, "source": [ - "Note that the values in the `meal_type` variable has stayed exactly the same, but the type of variable has changed from the object to category.\n", + "Note that the values in the `meal_plan` variable has stayed exactly the same, but the type of variable has changed from the object to category.\n", "\n", - "It is a bit tedious to have to go through columns one-by-one as single line assignments to apply type. An alternative is to pass a dictionary that maps column names into types, like follows:" + "It is a bit tedious to have to go through columns one-by-one as single line assignments to apply type. An alternative is to pass a dictionary that maps column names into types, like follows:\n" ] }, { @@ -228,8 +207,10 @@ "metadata": {}, "outputs": [], "source": [ - "students = students.astype({\"student_id\": \"int\", \"full_name\": \"string\", \"age\": \"int\"})\n", - "students.info()" + "students = students.cast(\n", + " {\"student_id\": pl.Int32, \"full_name\": pl.String, \"age\": pl.Int64}\n", + ")\n", + "students.schema" ] }, { @@ -239,7 +220,7 @@ "source": [ "### Exercises\n", "\n", - "1. What function would you use to read a file where fields were separated with \"\\|\"?" + "1. What function would you use to read a file where fields were separated with \"\\|\"?\n" ] }, { @@ -247,7 +228,7 @@ "id": "3fc7dbd6", "metadata": {}, "source": [ - "## Reading data from multiple files" + "## Reading data from multiple files\n" ] }, { @@ -257,7 +238,7 @@ "source": [ "Sometimes your data is split across multiple files instead of being contained in a single file. For example, you might have sales data for multiple months, with each month's data in a separate file: `01-sales.csv` for January, `02-sales.csv` for February, and `03-sales.csv` for March.\n", "\n", - "With `pd.read_csv()` you can read these data in one-by-one and then stack them on top of each other in a single data frame using the `pd.concat()` function. This looks like:\n" + "With `pl.read_csv()` you can read these data in one-by-one and then stack them on top of each other in a single data frame using the `pl.concat()` function. This looks like:\n" ] }, { @@ -268,10 +249,10 @@ "outputs": [], "source": [ "list_of_dataframes = [\n", - " pd.read_csv(x)\n", + " pl.read_csv(x)\n", " for x in [\"data/01-sales.csv\", \"data/02-sales.csv\", \"data/03-sales.csv\"]\n", "]\n", - "sales_files = pd.concat(list_of_dataframes)\n", + "sales_files = pl.concat(list_of_dataframes)\n", "sales_files" ] }, @@ -280,7 +261,7 @@ "id": "ea3904ca", "metadata": {}, "source": [ - "If you have many files you want to read in, it can get cumbersome to write out their names as a list. Instead, you can use the **glob** package (which is built in to Python) to find the files for you by matching a pattern in the file names. Note that there may be other CSV files in the directory data/, so here we specified `\"*-sales.csv\"` to ensure we only get those files that include the word sales. Here, `\"*\"` plays the role of a wildcard: it represents any series of characters." + "If you have many files you want to read in, it can get cumbersome to write out their names as a list. Instead, you can use the **glob** package (which is built in to Python) to find the files for you by matching a pattern in the file names. Note that there may be other CSV files in the directory data/, so here we specified `\"*-sales.csv\"` to ensure we only get those files that include the word sales. Here, `\"*\"` plays the role of a wildcard: it represents any series of characters.\n" ] }, { @@ -295,7 +276,7 @@ "list_of_csvs = glob.glob(\"data/*-sales.csv\")\n", "print(\"List of csvs is:\")\n", "print(list_of_csvs, \"\\n\")\n", - "sales_files = pd.concat([pd.read_csv(x) for x in list_of_csvs])\n", + "sales_files = pl.concat([pl.read_csv(x) for x in list_of_csvs])\n", "sales_files" ] }, @@ -306,9 +287,9 @@ "source": [ "## Writing to a file\n", "\n", - "Just as the typical pattern for reading files is `pd.read_FILETYPE()`, where filetype can be, for example, CSV, all of the ways of writing **pandas** data frames to disk have the pattern `DATAFRAME.to_FILETYPE()`. So to write our sales data to a CSV file, the code will be `sales_files.to_csv(FILEPATH)`, where filepath is the location plus name of the file you want to write to.\n", + "Just as the typical pattern for reading files is `pd.read_FILETYPE()`, where filetype can be, for example, CSV, all of the ways of writing **polars** data frames to disk have the pattern `DATAFRAME.write_FILETYPE()`. So to write our sales data to a CSV file, the code will be `sales_files.write_csv(FILEPATH)`, where filepath is the location plus name of the file you want to write to.\n", "\n", - "Let's see an example of writing data to file using our students data that we already did such good work on setting the data types for:" + "Let's see an example of writing data to file using our students data that we already did such good work on setting the data types for:\n" ] }, { @@ -318,7 +299,7 @@ "metadata": {}, "outputs": [], "source": [ - "students.to_csv(\"data/students-clean.csv\")" + "students.write_csv(\"data/students-clean.csv\")" ] }, { @@ -326,7 +307,7 @@ "id": "9c92b016", "metadata": {}, "source": [ - "Now let's read it back in and check the info on data types:" + "Now let's read it back in and check the info on data types:\n" ] }, { @@ -336,7 +317,7 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_csv(\"data/students-clean.csv\").info()" + "pl.read_csv(\"data/students-clean.csv\").schema" ] }, { @@ -344,13 +325,13 @@ "id": "164cd1c0", "metadata": {}, "source": [ - "Notice anything? We lost a lot of the nice data type work we did! While **pandas** guessed that some columns are integers, we lost the string and categorical variables. The reason for this is that plain text files are not able to carry any contextual information (though **pandas** will guess some column data types).\n", + "Notice anything? We lost a lot of the nice data type work we did! While **polars** guessed that some columns are integers, we lost the string and categorical variables. The reason for this is that plain text files are not able to carry any contextual information (though **polars** will guess some column data types).\n", "\n", - "If you want to save data in a file and have it remember the data types, you'll need to use a different data format. For temporary storage, we recommend using the *feather* format as it is very fast and interoperable with other programming languages. Interoperability is a good reason to avoid language-specific file formats such as Stata's .dta, R's .rds, and Python's .pickle.\n", + "If you want to save data in a file and have it remember the data types, you'll need to use a different data format. For temporary storage, we recommend using the _feather_ format as it is very fast and interoperable with other programming languages. Interoperability is a good reason to avoid language-specific file formats such as Stata's .dta, R's .rds, and Python's .pickle.\n", "\n", "Note that the feather format has an additional dependency in the form of a package called **pyarrow**. To install it, run `uv add pyarrow` in a terminal window.\n", "\n", - "Here's an example of writing to a feather file:" + "Here's an example of writing to a feather file:\n" ] }, { @@ -360,7 +341,7 @@ "metadata": {}, "outputs": [], "source": [ - "students.to_feather(\"data/students-clean.feather\")" + "students.write_ipc(\"data/students-clean.feather\")" ] }, { @@ -368,7 +349,7 @@ "id": "2119dd53", "metadata": {}, "source": [ - "Now let's re-open that feather file and take a look at the info attached to it." + "Now let's re-open that feather file and take a look at the info attached to it.\n" ] }, { @@ -378,7 +359,7 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_feather(\"data/students-clean.feather\").info()" + "pl.read_ipc(\"data/students-clean.feather\").schema" ] }, { @@ -390,7 +371,7 @@ "\n", "### Reading and Writing Other Data Formats\n", "\n", - "The image at the start of this chapter gives you a sense of what other formats are available but you can find a comprehensive list over at the [official **pandas** documentation on input and output](https://pandas.pydata.org/docs/user_guide/io.html)." + "You can find a comprehensive list of what formats are available over at the [official **polars** documentation on input and output](https://pandas.pydata.org/docs/user_guide/io.html).\n" ] }, { @@ -412,9 +393,6 @@ } ], "metadata": { - "interpreter": { - "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" - }, "jupytext": { "cell_metadata_filter": "-all", "encoding": "# -*- coding: utf-8 -*-", @@ -422,7 +400,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "python4ds", "language": "python", "name": "python3" }, @@ -436,7 +414,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/data-tidy.ipynb b/data-tidy.ipynb index e0ba7a7..157081f 100644 --- a/data-tidy.ipynb +++ b/data-tidy.ipynb @@ -485,7 +485,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/data-transform.ipynb b/data-transform.ipynb index 55f2df7..8dfc2c1 100644 --- a/data-transform.ipynb +++ b/data-transform.ipynb @@ -11,7 +11,7 @@ "\n", "It's very rare that data arrive in exactly the right form you need. Often, you'll need to create some new variables or summaries, or maybe you just want to rename the variables or reorder the observations to make the data a little easier to work with.\n", "\n", - "You'll learn how to do all that (and more!) in this chapter, which will introduce you to data transformation using the **pandas** package and a new dataset on flights that departed New York City in 2013.\n", + "You'll learn how to do all that (and more!) in this chapter, which will introduce you to data transformation using the **polars** package and a new dataset on flights that departed New York City in 2013.\n", "\n", "The goal of this chapter is to give you an overview of all the key tools for transforming a data frame, a special kind of object that holds tabular data.\n", "\n", @@ -19,7 +19,7 @@ "\n", "### Prerequisites\n", "\n", - "In this chapter we'll focus on the **pandas** package, one of the most widely used tools for data science. You'll need to ensure you have **pandas** installed. To do this, you can run" + "In this chapter we'll focus on the **polars** package, one of the most widely used tools for data science. You'll need to ensure you have **polars** installed. To do this, you can run\n" ] }, { @@ -29,7 +29,7 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd" + "import polars as pl" ] }, { @@ -37,9 +37,9 @@ "id": "438cc0a4", "metadata": {}, "source": [ - "If this command fails, you don't have **pandas** installed. Open up the terminal in Visual Studio Code (Terminal -> New Terminal), `cd` to the folder you are working in, and type in `uv add pandas`.\n", + "If this command fails, you don't have **polars** installed. Open up the terminal in Visual Studio Code (Terminal -> New Terminal), `cd` to the folder you are working in, and type in `uv add polars`.\n", "\n", - "Furthermore, if you wish to check which version of **pandas** you're using, it's" + "Furthermore, if you wish to check which version of **polars** you're using, it's\n" ] }, { @@ -49,7 +49,7 @@ "metadata": {}, "outputs": [], "source": [ - "pd.__version__" + "pl.__version__" ] }, { @@ -57,11 +57,11 @@ "id": "0c5e5b82", "metadata": {}, "source": [ - "You'll also need the data. Most of the time, data will need to be loaded from a file or the internet. These data are no different, but one of the amazing things about **pandas** is how many different types of data it can load, including from files on the internet.\n", + "You'll also need the data. Most of the time, data will need to be loaded from a file or the internet. These data are no different, but one of the amazing things about **polars** is how many different types of data it can load, including from files on the internet.\n", "\n", "The data is around 50MB in size so you will need a good internet connection or a little patience for it to download.\n", "\n", - "Let's download the data:" + "Let's download the data:\n" ] }, { @@ -71,8 +71,20 @@ "metadata": {}, "outputs": [], "source": [ + "import io\n", + "\n", + "import requests\n", + "\n", "url = \"https://raw.githubusercontent.com/byuidatascience/data4python4ds/master/data-raw/flights/flights.csv\"\n", - "flights = pd.read_csv(url)" + "resp = requests.get(url, timeout=60)\n", + "resp.raise_for_status()\n", + "\n", + "flights = pl.read_csv(\n", + " io.BytesIO(resp.content),\n", + " null_values=[\"NA\"],\n", + " truncate_ragged_lines=True,\n", + " ignore_errors=True,\n", + ")" ] }, { @@ -80,7 +92,7 @@ "id": "2907635c", "metadata": {}, "source": [ - "If the above code worked, then you've downloaded the data in CSV format and put it in a data frame. Let's look at the first few rows using the `.head()` function that works on all **pandas** data frames." + "If the above code worked, then you've downloaded the data in CSV format and put it in a data frame. Let's look at the first few rows using the `.head()` function that works on all **polars** data frames.\n" ] }, { @@ -98,7 +110,7 @@ "id": "68aada55", "metadata": {}, "source": [ - "To get more general information on the columns, the data types (`dtypes`) of the columns, and the size of the dataset, use `.info()`." + "To get more general information on the columns, the data types (`dtypes`) of the columns, and the size of the dataset, use `.glimpse()`.\n" ] }, { @@ -108,7 +120,7 @@ "metadata": {}, "outputs": [], "source": [ - "flights.info()" + "flights.glimpse(max_items_per_column=5)" ] }, { @@ -116,22 +128,23 @@ "id": "100189b8", "metadata": {}, "source": [ - "You might have noticed the short abbreviations that appear in the `Dtypes` column. These tell you the type of the values in their respective columns: `int64` is short for integer (eg whole numbers) and `float64` is short for double-precision floating point number (these are real numbers). `object` is a bit of a catch all category for any data type that **pandas** is not really confident about inferring. Although not found here, other data types include `string` for text and `datetime` for combinations of a date and time.\n", + "You might have noticed the short abbreviations that appear in the `dtype` column. These tell you the type of the values in their respective columns: `i64` is short for integer (eg whole numbers) and `f64` is short for double-precision floating point number (these are real numbers). **polars** has an `object` data type which allows storing arbitrary Python objects, but this makes you lose performance benefits, as **polars** is **strictly typed**. Although not found here, other data types include `str` for text and `datetime` for combinations of a date and time.\n", "\n", "The table below gives some of the most common data types you are likely to encounter.\n", "\n", - "| **Name of data type** | **Type of data** |\n", - "|:----------:|:-------------:|\n", - "| float64 | real numbers |\n", - "| category | categories |\n", - "| datetime64 | date times |\n", - "| int64 | integers |\n", - "| bool | True or False |\n", - "| string | text |\n", + "| **Name of data type** | **Type of data** |\n", + "| :-------------------: | :--------------: |\n", + "| Float64 | real numbers |\n", + "| Categorical | categories |\n", + "| Datetime | date and time |\n", + "| Date | date |\n", + "| Int64 | integers |\n", + "| Boolean | True or False |\n", + "| String | text |\n", "\n", "The different column data types are important because the operations you can perform on a column depend so much on its \"type\"; for example, you can remove all punctuation from strings while you can multiply ints and floats.\n", "\n", - "We would like to work with the `\"time_hour\"` variable in the form of a datetime; fortunately, **pandas** makes it easy to perform that conversion on that specific column" + "We would like to work with the `\"time_hour\"` variable in the form of a datetime; fortunately, **polars** makes it easy to perform that conversion on that specific column\n" ] }, { @@ -141,7 +154,7 @@ "metadata": {}, "outputs": [], "source": [ - "flights[\"time_hour\"]" + "flights.get_column(\"time_hour\")" ] }, { @@ -151,7 +164,7 @@ "metadata": {}, "outputs": [], "source": [ - "flights[\"time_hour\"] = pd.to_datetime(flights[\"time_hour\"], format=\"%Y-%m-%dT%H:%M:%SZ\")" + "flights.with_columns(pl.col(\"time_hour\").str.to_datetime())" ] }, { @@ -159,17 +172,17 @@ "id": "6dc43cee", "metadata": {}, "source": [ - "## **pandas** basics\n", + "## **polars** basics\n", "\n", - "**pandas** is a really comprehensive package, and this book will barely scratch the surface of what it can do. But it's built around a few simple ideas that, once they've clicked, make life a lot easier.\n", + "**polars** is a really comprehensive package, and this book will barely scratch the surface of what it can do. But it's built around a few simple ideas that, once they've clicked, make life a lot easier.\n", "\n", - "Let’s start with the absolute basics. The most basic pandas object is DataFrame. A DataFrame is a 2-dimensional data structure that can store data of different types (including characters, integers, floating point values, categorical data, even lists) in columns. It is made up of rows and columns (with each row-column cell containing a value), plus two bits of contextual information: the index (which carries information about each row) and the column names (which carry information about each column).\n", + "Let’s start with the absolute basics. The most basic polars object is DataFrame. A DataFrame is a 2-dimensional data structure that can store data of different types (including characters, integers, floating point values, categorical data, even lists) in columns. It is made up of rows and columns (with each row-column cell containing a value), plus contextual information: column names (which carry information about each column).\n", "\n", - "![](https://pandas.pydata.org/docs/_images/01_table_dataframe.svg)\n", - "\n", - "Perhaps the most important notion to have about **pandas** data frames is that they are built around an index that sits on the left-hand side of the data frame. Every time you perform an operation on a data frame, you need to think about how it might or might not affect the index; or, put another way, whether you want to modify the index.\n", + "::: {.callout-note}\n", + "Note: If you're coming from **pandas**, be aware that **polars** does not use an index column and each row is indexed by its integer position in the table.\n", + ":::\n", "\n", - "Let's see a simple example of this with a made-up data frame:" + "![](./dataframe_illustration.svg)\n" ] }, { @@ -179,7 +192,7 @@ "metadata": {}, "outputs": [], "source": [ - "df = pd.DataFrame(\n", + "df = pl.DataFrame(\n", " data={\n", " \"col0\": [0, 0, 0, 0],\n", " \"col1\": [0, 0, 0, 0],\n", @@ -187,7 +200,6 @@ " \"col3\": [\"a\", \"b\", \"b\", \"a\"],\n", " \"col4\": [\"alpha\", \"gamma\", \"gamma\", \"gamma\"],\n", " },\n", - " index=[\"row\" + str(i) for i in range(4)],\n", ")\n", "df.head()" ] @@ -197,7 +209,7 @@ "id": "185ba56e", "metadata": {}, "source": [ - "You can see there are 5 columns (named `\"col0\"` to `\"col4\"`) and that the index consists of four entries named `\"row0\"` to `\"row3\"`." + "You can see there are 5 columns (named `\"col0\"` to `\"col4\"`).\n" ] }, { @@ -205,14 +217,14 @@ "id": "3f325661", "metadata": {}, "source": [ - "A second key point you should know is that the operations on a **pandas** data frame can be chained together. We need not perform one assignment per line of code; we can actually do multiple assignments in a single command.\n", + "A second key point you should know is that the operations on a **polars** data frame can be chained together. We need not perform one assignment per line of code; we can actually do multiple assignments in a single command.\n", "\n", "Let's see an example of this. We're going to string together four operations:\n", "\n", - "1. we will use `query()` to find only the rows where the destination `\"dest\"` column has the value `\"IAH\"`. This doesn't change the index, it only removes irrelevant rows. In effect, this step removes rows we're not interested in.\n", - "2. we will use `groupby()` to group rows by the year, month, and day (we pass a list of columns to the `groupby()` function). This step changes the index; the new index will have three columns in that track the year, month, and day. In effect, this step changes the index.\n", - "3. we will choose which columns we wish to keep after the `groupby()` operation by passing a list of them to a set of square brackets (the double brackets are because it's a list within a data frame). Here we just want one column, `\"arr_delay\"`. This doesn't affect the index. In effect, this step removes columns we're not interested in.\n", - "4. finally, we must specify what `groupby()` operation we wish to apply; when aggregating the information in multiple rows down to one row, we need to say how that information should be aggregated. In this case, we'll use the `mean()`. In effect, this step applies a statistic to the variable(s) we selected earlier, across the groups we created earlier." + "1. we will use `filter()` to find only the rows where the destination `\"dest\"` column has the value `\"IAH\"`. We use the `pl.col()` expression to select the column for filtering condition. In effect, this step removes rows we're not interested in.\n", + "2. we will use `group_by()` to group rows by the year, month, and day (we pass a list of columns to the `group_by()` function).\n", + "3. we will choose which columns to perform aggregation on after the `group_by()` operation by using the `pl.col()` expression inside `agg()`, to select the column. Here we just want one column, `\"arr_delay\"`. In effect, this step removes columns we're not interested in.\n", + "4. finally, we must specify what `agg()` operation we wish to apply; when aggregating the information in multiple rows down to one row, we need to say how that information should be aggregated. In this case, we'll use the `mean()`. In effect, this step applies a statistic to the variable(s) we selected earlier, across the groups we created earlier.\n" ] }, { @@ -222,7 +234,9 @@ "metadata": {}, "outputs": [], "source": [ - "(flights.query(\"dest == 'IAH'\").groupby([\"year\", \"month\", \"day\"])[[\"arr_delay\"]].mean())" + "flights.filter(pl.col(\"dest\") == \"IAH\").group_by([\"year\", \"month\", \"day\"]).agg(\n", + " pl.col(\"arr_delay\").mean()\n", + ")" ] }, { @@ -230,16 +244,15 @@ "id": "b8b85551", "metadata": {}, "source": [ - "You can see here that we've created a new data frame with a new index. To do it, we used four key operations:\n", + "You can see here that we've created a new data frame. To do it, we used three key operations:\n", "\n", "1. manipulating rows\n", - "2. manipulating the index\n", - "3. manipulating columns\n", - "4. applying statistics\n", + "2. manipulating columns\n", + "3. applying statistics\n", "\n", "Most operations you could want to do to a single data frame are covered by these, but there are different options for each of them depending on what you need.\n", "\n", - "Let's now dig a bit more into these operations." + "Let's now dig a bit more into these operations.\n" ] }, { @@ -249,7 +262,7 @@ "source": [ "## Manipulating Rows in Data Frames\n", "\n", - "Let's create some fake data to show how this works." + "Let's create some fake data to show how this works.\n" ] }, { @@ -261,13 +274,13 @@ "source": [ "import numpy as np\n", "\n", - "df = pd.DataFrame(\n", + "df = pl.DataFrame(\n", " data=np.reshape(range(36), (6, 6)),\n", - " index=[\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"],\n", - " columns=[\"col\" + str(i) for i in range(6)],\n", - " dtype=float,\n", + " schema=[\"col\" + str(i) for i in range(6)],\n", + ")\n", + "df.insert_column(\n", + " 6, pl.Series(\"col6\", [\"apple\", \"orange\", \"pineapple\", \"mango\", \"kiwi\", \"lemon\"])\n", ")\n", - "df[\"col6\"] = [\"apple\", \"orange\", \"pineapple\", \"mango\", \"kiwi\", \"lemon\"]\n", "df" ] }, @@ -278,9 +291,10 @@ "source": [ "### Accessing Rows\n", "\n", - "To access a particular row directly, you can use `df.loc['rowname']` or `df.loc[['rowname1', 'rowname2']]` for two different rows.\n", + "To access a particular row directly, you can get that by index (location in the data) or predicate/expression using `.row()`, which returns as a tuple.\n", + "Remember that Python indices begin from zero, so to retrieve the first row by index you would use `.row(0)`:\n", "\n", - "For example," + "For example,\n" ] }, { @@ -290,7 +304,11 @@ "metadata": {}, "outputs": [], "source": [ - "df.loc[[\"a\", \"b\"]]" + "# Gets the first row of the DataFrame\n", + "df.row(0)\n", + "\n", + "# Gets the fifth row of the DataFrame\n", + "df.row(4)" ] }, { @@ -298,7 +316,7 @@ "id": "18124edd", "metadata": {}, "source": [ - "But you can also access particular rows based on their location in the data frame using `.iloc`. Remember that Python indices begin from zero, so to retrieve the first row you would use `.iloc[0]`:\n" + "We can also access particular rows based on a predicate using `.row()` with the `by_predicate` parameter.\n" ] }, { @@ -308,7 +326,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.iloc[0]" + "df.row(by_predicate=pl.col(\"col6\") == \"mango\")" ] }, { @@ -316,7 +334,7 @@ "id": "ca822472", "metadata": {}, "source": [ - "This works for multiple rows too. Let's grab the first and third rows (in positions 0 and 2) by passing a list of positions:" + "To get the row as a dictionary instead of a tuple with a mapping of column names to row values, specify `named=True`\n" ] }, { @@ -326,15 +344,29 @@ "metadata": {}, "outputs": [], "source": [ - "df.iloc[[0, 2]]" + "# Get the first row of the DataFrame as a dictionary\n", + "df.row(0, named=True)\n", + "\n", + "# Get the row where col6 is \"mango\" as a dictionary\n", + "df.row(by_predicate=pl.col(\"col6\") == \"mango\", named=True)" ] }, { "cell_type": "markdown", - "id": "381eb34d", + "id": "980b7be6", "metadata": {}, "source": [ - "There are other ways to access multiple rows that make use of *slicing* but we'll leave that topic for another time." + "We can also access rows using the `.slice()` function. As the function name implies, we get a slice of the DataFrame. we can use this to get a single row or a number of rows. To use this, we give it an offset - a start index, negative indexing is supported to index from the bottom of the DataFrame - and a length of the slice. This returns a DataFrame\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f7e8e892", + "metadata": {}, + "outputs": [], + "source": [ + "df.slice(-2, 2)" ] }, { @@ -342,9 +374,9 @@ "id": "77f67ac2", "metadata": {}, "source": [ - "### Filtering rows with query\n", + "### Filtering rows\n", "\n", - "As with the flights example, we can also filter rows based on a condition using `query()`:" + "As with the flights example, we can also filter rows based on a condition using `filter()`:\n" ] }, { @@ -354,7 +386,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.query(\"col6 == 'kiwi' or col6 == 'pineapple'\")" + "df.filter((pl.col(\"col6\") == \"kiwi\") | (pl.col(\"col6\") == \"pineapple\"))" ] }, { @@ -362,7 +394,7 @@ "id": "000eb292", "metadata": {}, "source": [ - "For numbers, you can also use the greater than and less than signs:" + "For numbers, you can also use the greater than and less than signs:\n" ] }, { @@ -372,7 +404,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.query(\"col0 > 6\")" + "df.filter(pl.col(\"col0\") > 6)" ] }, { @@ -380,7 +412,7 @@ "id": "f5e03f63", "metadata": {}, "source": [ - "In fact, there are lots of options that work with `query()`: as well as `>` (greater than), you can use `>=` (greater than or equal to), `<` (less than), `<=` (less than or equal to), `==` (equal to), and `!=` (not equal to). You can also use the commands `and` as well as `or` to combine multiple conditions. Here's an example of `and` from the `flights` data frame:" + "In fact, there are lots of options that work with `filter()`: as well as `>` (greater than), you can use `>=` (greater than or equal to), `<` (less than), `<=` (less than or equal to), `==` (equal to), and `!=` (not equal to). You can also use operators `&` as well as `|` to combine multiple conditions. Here's an example of `&` from the `flights` data frame:\n" ] }, { @@ -391,7 +423,7 @@ "outputs": [], "source": [ "# Flights that departed on January 1\n", - "flights.query(\"month == 1 and day == 1\")" + "flights.filter((pl.col(\"month\") == 1) & (pl.col(\"day\") <= 5))" ] }, { @@ -399,7 +431,7 @@ "id": "bd0af6fc", "metadata": {}, "source": [ - "Note that equality is tested by `==` and *not* by `=`, because the latter is used for assignment." + "Note that equality is tested by `==` and _not_ by `=`, because the latter is used for assignment.\n" ] }, { @@ -409,7 +441,17 @@ "source": [ "### Re-arranging Rows\n", "\n", - "Again and again, you will want to re-order the rows of your data frame according to the values in a particular column. **pandas** makes this very easy via the `.sort_values()` function. It takes a data frame and a set of column names to sort by. If you provide more than one column name, each additional column will be used to break ties in the values of preceding columns. For example, the following code sorts by the departure time, which is spread over four columns." + "Again and again, you will want to re-order the rows of your data frame according to the values in a particular column. **polars** makes this very easy via the `.sort()` function. You can sort by single or multiple column names and also by expressions. If you provide more than one column name, each additional column will be used to break ties in the values of preceding columns. For example, the following code sorts by the departure time, which is spread over four columns.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "395c9c62", + "metadata": {}, + "outputs": [], + "source": [ + "flights.sort(\"dep_time\")" ] }, { @@ -419,7 +461,11 @@ "metadata": {}, "outputs": [], "source": [ - "flights.sort_values([\"year\", \"month\", \"day\", \"dep_time\"])" + "# Sort by multiple columns by passing a list of columns.\n", + "flights.sort([\"year\", \"month\", \"day\", \"dep_time\"])\n", + "\n", + "# Or use positional arguments to sort by multiple columns in the same way.\n", + "flights.sort(\"year\", \"month\", \"day\", \"dep_time\")" ] }, { @@ -427,8 +473,8 @@ "id": "39a6e9b1", "metadata": {}, "source": [ - "You can use the keyword argument `ascending=False` to re-order by a column or columns in descending order.\n", - "For example, this code shows the most delayed flights:" + "You can use the keyword argument `descending=True` to re-order by a column or columns in descending order.\n", + "For example, this code shows the most delayed flights:\n" ] }, { @@ -438,7 +484,17 @@ "metadata": {}, "outputs": [], "source": [ - "flights.sort_values(\"dep_delay\", ascending=False)" + "flights.sort(\"dep_delay\", descending=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80bf3df7", + "metadata": {}, + "outputs": [], + "source": [ + "flights.sort([\"dep_delay\", \"arr_delay\"], descending=[True, False])" ] }, { @@ -447,7 +503,7 @@ "metadata": {}, "source": [ "You can of course combine all of the above row manipulations to solve more complex problems.\n", - "For example, we could look for the top three destinations of the flights that were most delayed on arrival that left on roughly on time:" + "For example, we could look for the top three destinations of the flights that were most delayed on arrival that left on roughly on time:\n" ] }, { @@ -458,9 +514,9 @@ "outputs": [], "source": [ "(\n", - " flights.query(\"dep_delay <= 10 and dep_delay >= -10\")\n", - " .sort_values(\"arr_delay\", ascending=False)\n", - " .iloc[[0, 1, 2]]\n", + " flights.filter((pl.col(\"dep_delay\") <= 10) & (pl.col(\"dep_delay\") >= -10))\n", + " .sort(\"arr_delay\", descending=True)\n", + " .head(3)\n", ")" ] }, @@ -485,13 +541,13 @@ "\n", " f. Were delayed by at least an hour, but made up over 30 minutes in flight\n", "\n", - "2. Sort `flights` to find the flights with longest departure delays.\n", + "2. Sort `flights` to find the flights with longest departure delays.\n", "\n", - "3. Sort `flights` to find the fastest flights\n", + "3. Sort `flights` to find the fastest flights\n", "\n", - "4. Which flights traveled the farthest?\n", + "4. Which flights traveled the farthest?\n", "\n", - "5. Does it matter what order you used `query()` and `sort_values()` in if you're using both? Why/why not? Think about the results and how much work the functions would have to do." + "5. Does it matter what order you used `filter()` and `sort()` in if you're using both? Why/why not? Think about the results and how much work the functions would have to do.\n" ] }, { @@ -501,11 +557,7 @@ "source": [ "## Manipulating Columns\n", "\n", - "This section will show you how to apply various operations you may need to columns in your data frame.\n", - "\n", - "::: {.callout-note}\n", - "Some **pandas** operations can apply either to columns or rows, depending on the syntax used. For example, accessing values by position can be achieved in the same way for rows and columns via `.iloc` where to access the ith row you would use `df.iloc[i]` and to access the jth column you would use `df.iloc[:, j]` where `:` stands in for 'any row'.\n", - ":::" + "This section will show you how to apply various operations you may need to columns in your data frame.\n" ] }, { @@ -515,7 +567,7 @@ "source": [ "### Creating New Columns\n", "\n", - "Let's now move on to creating new columns, either using new information or from existing columns. Given a data frame, `df`, creating a new column with the same value repeated is as easy as using square brackets with a string (text enclosed by quotation marks) in." + "Let's now move on to creating new columns, either using new information or from existing columns. Given a data frame, `df`, creating a new column with the same value repeated is done by using `.with_columns()`, with an expression assigned to a column name. Here we use `pl.lit()` which returns an expression representing a literal value, 5 in our case.\n" ] }, { @@ -525,7 +577,7 @@ "metadata": {}, "outputs": [], "source": [ - "df[\"new_column0\"] = 5\n", + "df = df.with_columns(new_column0=pl.lit(5))\n", "df" ] }, @@ -534,7 +586,7 @@ "id": "55bc84a5", "metadata": {}, "source": [ - "If we do the same operation again, but with a different right-hand side, it will overwrite what was already in that column. Let's see this with an example where we put different values in each position by assigning a list to the new column." + "If we do the same operation again, but this time assigning a **_Series_** to the same column, it will overwrite what was already in that column. A **_Series_** repesents a single column in a Polars DataFrame. Let's see this with an example where we put different values in each position by assigning a list to the new column.\n" ] }, { @@ -544,7 +596,7 @@ "metadata": {}, "outputs": [], "source": [ - "df[\"new_column0\"] = [0, 1, 2, 3, 4, 5]\n", + "df = df.with_columns(new_column0=pl.Series([0, 1, 2, 3, 4, 5]))\n", "df" ] }, @@ -557,7 +609,7 @@ "What happens if you try to use assignment where the right-hand side values are longer or shorter than the length of the data frame?\n", ":::\n", "\n", - "By passing a list within the square brackets, we can actually create more than one new column:" + "We can actually use `.with_columns` to create more than one new column:\n" ] }, { @@ -567,7 +619,7 @@ "metadata": {}, "outputs": [], "source": [ - "df[[\"new_column1\", \"new_column2\"]] = [5, 6]\n", + "df = df.with_columns(new_column1=pl.lit(5), new_column2=pl.lit(6))\n", "df" ] }, @@ -576,7 +628,7 @@ "id": "10792ddd", "metadata": {}, "source": [ - "Very often, you will want to create a new column that is the result of an operation on existing columns. There are a couple of ways to do this. The 'stand-alone' method works in a similar way to what we've just seen except that we refer to the data frame on the right-hand side of the assignment statement too:" + "Very often, you will want to create a new column or modify a column that is the result of an operation on existing columns. There are a couple of ways to do this. The 'stand-alone' method works in a similar way to what we've just seen except that we refer to the data frame on the right-hand side of the assignment statement too:\n" ] }, { @@ -586,8 +638,7 @@ "metadata": {}, "outputs": [], "source": [ - "df[\"new_column3\"] = df[\"col0\"] - df[\"new_column0\"]\n", - "df" + "df.with_columns(new_column2=pl.col(\"col0\") - pl.col(\"new_column0\"))" ] }, { @@ -595,7 +646,8 @@ "id": "03172fa9", "metadata": {}, "source": [ - "The other way to do this involves an 'assign()' statement and is used when you wish to chain multiple steps together (like we saw earlier). These use a special syntax called a 'lambda' statement, which (here at least) just provides a way of specifying to **pandas** that we wish to perform the operation on every row. Below is an example using the flights data. You should note though that the word 'row' below is a dummy; you could replace it with any variable name (for example, `x`) but `row` makes what is happening a little bit clearer." + "We can use `.alias()` with an expression to assign column names, when creating new columns.\n", + "We can chain multiple expressions together with `.with_columns()`, which would create multiple new columns with the names assigned to `.alias()`.\n" ] }, { @@ -605,36 +657,18 @@ "metadata": {}, "outputs": [], "source": [ - "(\n", - " flights.assign(\n", - " gain=lambda row: row[\"dep_delay\"] - row[\"arr_delay\"],\n", - " speed=lambda row: row[\"distance\"] / row[\"air_time\"] * 60,\n", - " )\n", + "flights.with_columns(\n", + " (pl.col(\"dep_delay\") - pl.col(\"arr_delay\")).alias(\"gain\"),\n", + " (pl.col(\"distance\") / pl.col(\"air_time\") * 60).alias(\"speed\"),\n", ")" ] }, - { - "cell_type": "markdown", - "id": "c531df3e", - "metadata": {}, - "source": [ - "::: {.callout-note}\n", - "A lambda function is like any normal function in Python except that it has no name, and it tends to be contained in one line of code. A lambda function is made of an argument, a colon, and an expression, like the following lambda function that multiplies an input by three.\n", - "\n", - "```python\n", - "lambda x: x*3\n", - "```\n", - "\n", - ":::" - ] - }, { "cell_type": "markdown", "id": "82a97330", "metadata": {}, "source": [ - "### Accessing Columns\n", - "\n" + "### Accessing Columns\n" ] }, { @@ -642,7 +676,7 @@ "id": "7599db58", "metadata": {}, "source": [ - "Just as with selecting rows, there are many options and ways to select the columns to operate on. The one with the simplest syntax is the name of the data frame followed by square brackets and the column name (as a string)" + "Just as with selecting rows, there are many options and ways to select the columns to operate on. The one with the simplest syntax is the name of the data frame followed by square brackets and the column name (as a string)\n" ] }, { @@ -660,7 +694,7 @@ "id": "63bca028", "metadata": {}, "source": [ - "If you need to select *multiple* columns, you cannot just pass a string into `df[...]`; instead you need to pass an object that is iterable (and so have multiple items). The most straight forward way to select multiple columns is to pass a *list*. Remember, lists comes in square brackets so we're going to see something with repeated square brackets: one for accessing the data frame's innards and one for the list." + "If you need to select _multiple_ columns, you cannot just pass a string into `df[...]`; instead you need to pass an object that is iterable (and so have multiple items). The most straight forward way to select multiple columns is to pass a _list_. Remember, lists comes in square brackets so we're going to see something with repeated square brackets: one for accessing the data frame's innards and one for the list.\n" ] }, { @@ -675,46 +709,63 @@ }, { "cell_type": "markdown", - "id": "2b2a7be0", + "id": "a6fdfc17", "metadata": {}, "source": [ - "If you want to access particular rows at the same time, use the `.loc` access function:" + "We can also use `.select()` on the data frame to select columns, passing a single string to select a single column or an iterable, like a _list_, _positional arguments_ or _keyword arguments_, to select multiple columns. **Using _keyword arguments_ renames the columns in the output**\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "eabfd313", + "id": "1bc0cd22", "metadata": {}, "outputs": [], "source": [ - "df.loc[[\"a\", \"b\"], [\"col0\", \"new_column0\", \"col2\"]]" + "# selecting a single column\n", + "df.select(\"col0\")\n", + "\n", + "# Using positional arguments to select multiple columns\n", + "df.select(\"col0\", \"new_column0\", \"col2\")\n", + "\n", + "# Using keyword arguments to rename columns in the output\n", + "df.select(col1=\"col0\", col2=\"new_column0\", col3=\"col2\")" ] }, { "cell_type": "markdown", - "id": "c1b7db13", + "id": "a806be16", "metadata": {}, "source": [ - "And, just as with rows, we can access columns by their position using `.iloc` (where `:` stands in for 'any row')." + "Expressions are also accepted\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "b6ae1605", + "id": "ed447fb7", "metadata": {}, "outputs": [], "source": [ - "df.iloc[:, [0, 1]]" + "df.select(pl.col(\"col0\"), pl.col(\"new_column0\") + 2, pl.col(\"col2\") * 2)" ] }, { "cell_type": "markdown", - "id": "509dc236", + "id": "2b2a7be0", + "metadata": {}, + "source": [ + "If we want to access particular rows at the same time, we can chain `.filter()` or `.slice()` to the `.select()` function:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eabfd313", "metadata": {}, + "outputs": [], "source": [ - "There are other ways to access multiple columns that make use of slicing but we’ll leave that topic for another time." + "df.select(\"col0\", \"new_column0\", \"col2\").slice(0, 2)" ] }, { @@ -722,46 +773,55 @@ "id": "17b928c8", "metadata": {}, "source": [ - "Sometimes, you'll want to select columns based on the *type* of data that they hold. For this, **pandas** provides a function `.select_dtypes()`. Let's use this to select all columns with integers in the flights data." + "Sometimes, we'll want to select columns based on the _type_ of data that they hold. For this, we can call **polars** data types with an expression inside `.select()`. Let's use this to select all columns with integers in the flights data.\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "62f578d1", + "id": "aed67406", "metadata": {}, "outputs": [], "source": [ - "flights.select_dtypes(\"int\")" + "flights.select(pl.col(pl.Int64))" ] }, { "cell_type": "markdown", - "id": "9aec778c", + "id": "8cb930af", "metadata": {}, "source": [ - "There are other occasions when you'd like to select columns based on criteria such as patterns in the *name* of the column. Because Python has very good support for text, this is very possible but doesn't tend to be so built-in to **pandas** functions. The trick is to generate a list of column names that you want from the pattern you're interested in.\n", - "\n", - "Let's see a couple of examples. First, let's get all columns in our `df` data frame that begin with `\"new_...\"`. We'll generate a list of true and false values reflecting if each of the columns begins with \"new\" and then we'll pass those true and false values to `.loc`, which will only give columns for which the result was `True`. To show what's going on, we'll break it into two steps:" + "**polars** also provides a `selectors` module that we can use to select columns based on both data types and criteria such as patterns in the name of the column.\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "5aaae8bd", + "id": "62f578d1", "metadata": {}, "outputs": [], "source": [ - "print(\"The list of columns:\")\n", - "print(df.columns)\n", - "print(\"\\n\")\n", + "import polars.selectors as S\n", "\n", - "print(\"The list of true and false values:\")\n", - "print(df.columns.str.startswith(\"new\"))\n", - "print(\"\\n\")\n", + "# Select all integer columns\n", + "flights.select(S.integer())\n", + "\n", + "# Exclude string columns\n", + "flights.select(S.exclude(S.string()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4e486db", + "metadata": {}, + "outputs": [], + "source": [ + "# Select columns that contain \"delay\" in their name\n", + "flights.select(S.contains(\"delay\"))\n", "\n", - "print(\"The selection from the data frame:\")\n", - "df.loc[:, df.columns.str.startswith(\"new\")]" + "# Select columns that start with \"arr\"\n", + "flights.select(S.starts_with(\"arr\"))" ] }, { @@ -769,7 +829,7 @@ "id": "b514cbf4", "metadata": {}, "source": [ - "As well as `startswith()`, there are other commands like `endswith()`, `contains()`, `isnumeric()`, and `islower()`." + "Other `selectors` commands include `ends_with()`, `by_index()`, `first()`, `last()`, `duration()`, `numeric()`, `boolean()` and more\n" ] }, { @@ -779,7 +839,7 @@ "source": [ "### Renaming Columns\n", "\n", - "There are three easy ways to rename columns, depending on what the context is. The first is to use the dedicated `rename()` function with an object called a dictionary. Dictionaries in Python consist of curly brackets with comma separated pairs of values where the first values maps into the second value. An example of a dictionary would be `{'old_col1': 'new_col1', 'old_col2': 'new_col2'}`. Let's see this in practice (but note that we are not 'saving' the resulting data frame, just showing it—to save it, you'd need to add `df = ` to the left-hand side of the code below)." + "We use the dedicated `rename()` function with a mapping, such as a dictionary or a lambda function. Dictionaries in Python consist of curly brackets with comma separated pairs of values where the first values maps into the second value. An example of a dictionary would be `{'old_col1': 'new_col1', 'old_col2': 'new_col2'}`. Let's see this in practice (but note that we are not 'saving' the resulting data frame, just showing it—to save it, you'd need to add `df = ` to the left-hand side of the code below).\n" ] }, { @@ -789,7 +849,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.rename(columns={\"col3\": \"letters\", \"col4\": \"names\", \"col6\": \"fruit\"})" + "df.rename({\"col3\": \"letters\", \"col4\": \"names\", \"col6\": \"fruit\"})" ] }, { @@ -797,7 +857,7 @@ "id": "0a673852", "metadata": {}, "source": [ - "The second method is for when you want to rename all of the columns. For that you simply set `df.columns` equal to the new set of columns that you'd like to have. For example, we might want to capitalise the first letter of each column using `str.capitalize()` and assign that to `df.columns`." + "Using a lambda function, maps each column name as its argument, which you can then perform an operation on.\n" ] }, { @@ -807,26 +867,17 @@ "metadata": {}, "outputs": [], "source": [ - "df.columns = df.columns.str.capitalize()\n", - "df" + "df.rename(lambda column_name: column_name.upper())" ] }, { "cell_type": "markdown", - "id": "7a8b9660", - "metadata": {}, - "source": [ - "Finally, we might be interested in just replacing specific parts of column names. In this case, we can use `.str.replace()`. As an example, let's add the word `\"Original\"` ahead of the original columns:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3dd7606b", + "id": "aa80d44b", "metadata": {}, - "outputs": [], "source": [ - "df.columns.str.replace(\"Col\", \"Original_column\")" + "::: {.callout-tip}\n", + "A _lambda function_ is a small, anonymous function in Python that performs a single operation. It's a shorthand way to create a function without using `def`.\n", + ":::\n" ] }, { @@ -834,7 +885,7 @@ "id": "09632b99", "metadata": {}, "source": [ - "### Re-ordering Columns" + "### Re-ordering Columns\n" ] }, { @@ -844,9 +895,9 @@ "source": [ "By default, new columns are added to the right-hand side of the data frame. But you may have reasons to want the columns to appear in a particular order, or perhaps you'd just find it more convenient to have new columns on the left-hand side when there are many columns in a data frame (which happens a lot).\n", "\n", - "The simplest way to re-order (all) columns is to create a new list of their names with them in the order that you'd like them: but be careful you don't forget any columns that you'd like to keep! \n", + "The simplest way to re-order (all) columns is to create a new list of their names with them in the order that you'd like them: but be careful you don't forget any columns that you'd like to keep!\n", "\n", - "Let's see an example with a fresh version of the fake data from earlier. We'll put all of the odd-numbered columns first, in descending order, then the even similarly." + "Let's see an example with a fresh version of the fake data from earlier. We'll put all of the odd-numbered columns first, in descending order, then the even similarly using `.select()`.\n" ] }, { @@ -856,11 +907,8 @@ "metadata": {}, "outputs": [], "source": [ - "df = pd.DataFrame(\n", - " data=np.reshape(range(36), (6, 6)),\n", - " index=[\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"],\n", - " columns=[\"col\" + str(i) for i in range(6)],\n", - " dtype=float,\n", + "df = pl.DataFrame(\n", + " data=np.reshape(range(36), (6, 6)), schema=[\"col\" + str(i) for i in range(6)]\n", ")\n", "df" ] @@ -868,11 +916,11 @@ { "cell_type": "code", "execution_count": null, - "id": "b9a409ac", + "id": "3c2029cc", "metadata": {}, "outputs": [], "source": [ - "df = df[[\"col5\", \"col3\", \"col1\", \"col4\", \"col2\", \"col0\"]]\n", + "df = df.select([\"col5\", \"col3\", \"col1\", \"col4\", \"col2\", \"col0\"])\n", "df" ] }, @@ -881,7 +929,7 @@ "id": "dd91d87a", "metadata": {}, "source": [ - "Of course, this is quite tedious if you have lots of columns! There are methods that can help make this easier depending on your context. Perhaps you'd just liked to sort the columns in order? This can be achieved by combining `sorted()` and the `reindex()` command (which works for rows or columns) with `axis=1`, which means the second axis (i.e. columns)." + "Of course, this is quite tedious if you have lots of columns! There are methods that can help make this easier depending on your context. Perhaps you'd just liked to sort the columns in order? This can be achieved by combining `sorted()` and the `.select()` function, passing the DataFrame `.columns`.\n" ] }, { @@ -891,47 +939,11 @@ "metadata": {}, "outputs": [], "source": [ - "df.reindex(sorted(df.columns), axis=1)" - ] - }, - { - "cell_type": "markdown", - "id": "28e49605", - "metadata": {}, - "source": [ - "## Review of How to Access Rows, Columns, and Values\n", - "\n", - "With all of these different ways to access values in data frames, it can get confusing. These are the different ways to get the first column of a data frame (when that first column is called `column` and the data frame is `df`):\n", - "\n", - "- `df.column`\n", - "- `df[\"column\"]`\n", - "- `df.loc[:, \"column\"]`\n", - "- `df.iloc[:, 0]`\n", - "\n", - "Note that `:` means 'give me everything'! The ways to access rows are similar (here assuming the first row is called `row`):\n", - "\n", - "- `df.loc[\"row\", :]`\n", - "- `df.iloc[0, :]`\n", - "\n", - "And to access the first value (ie the value in first row, first column):\n", - "\n", - "- `df.column[0]`\n", - "- `df[\"column\"][0]`\n", - "- `df.iloc[0, 0]`\n", - "- `df.loc[\"row\", \"column\"]`\n", - "\n", - "In the above examples, square brackets are instructions about *where* to grab bits from the data frame. They are a bit like an address system for values within a data frame. Square brackets *also* denote lists though. So if you want to select *multiple* columns or rows, you might see syntax like this:\n", - "\n", - "`df.loc[[\"row0\", \"row1\"], [\"column0\", \"column2\"]]`\n", - "\n", - "which picks out two rows and two columns via the lists `[\"row0\", \"row1\"]` and `[\"column0\", \"column2\"]`. Because there are lists alongside the usual system of selecting values, there are two sets of square brackets.\n", - "\n", - "::: {.callout-tip title=\"Tip\"}\n", + "# Alphabetical order\n", + "df.select(sorted(df.columns))\n", "\n", - "If you only want to remember one syntax for accessing rows and columns by name, use the pattern `df.loc[[\"row0\", \"row1\", ...], [\"col0\", \"col1\", ...]]`. This also works with a single row or a single column (or both).\n", - "\n", - "If you only want to remember one syntax for accessing rows and columns by position, use the pattern `df.iloc[[0, 1, ...], [0, 1, ...]]`. This also works with a single row or a single column (or both).\n", - ":::\n" + "# Reverse alphabetical order\n", + "df.select(sorted(df.columns, reverse=True))" ] }, { @@ -941,29 +953,13 @@ "source": [ "### Column and Row Exercises\n", "\n", - "1. Compare `air_time` with `arr_time - dep_time`. What do you expect to see? What do you see What do you need to do to fix it?\n", + "1. Compare `air_time` with `arr_time - dep_time`. What do you expect to see? What do you see? What do you need to do to fix it?\n", "\n", "2. Compare `dep_time`, `sched_dep_time`, and `dep_delay`. How would you expect those three numbers to be related?\n", "\n", "3. Brainstorm as many ways as possible to select `dep_time`, `dep_delay`, `arr_time`, and `arr_delay` from `flights`.\n", "\n", - "4. What happens if you include the name of a row or column multiple times when trying to select them?\n", - "\n", - "5. What does the `.isin()` function do in the following?\n", - "\n", - " ```python\n", - " flights.columns.isin([\"year\", \"month\", \"day\", \"dep_delay\", \"arr_delay\"])\n", - " ```\n", - "\n", - "6. Does the result of running the following code surprise you?\n", - " How do functions like `str.contains` deal with case by default?\n", - " How can you change that default?\n", - "\n", - " ```python\n", - " flights.loc[:, flights.columns.str.contains(\"TIME\")]\n", - " ```\n", - "\n", - " (Hint: you can use help even on functions that apply to data frames, eg use `help(flights.columns.str.contains)`)" + "4. What happens if you include the name of a row or column multiple times when trying to select them?\n" ] }, { @@ -971,9 +967,9 @@ "id": "a3c837e4", "metadata": {}, "source": [ - "## Grouping, changing the index, and applying summary statistics\n", + "## Grouping and applying summary statistics\n", "\n", - "So far you've learned about working with rows and columns. **pandas** gets even more powerful when you add in the ability to work with groups. Creating groups will often also mean a change of index. And because groups tend to imply an aggregation or pooling of data, they often go hand-in-hand with the application of a summary statistic.\n", + "So far you've learned about working with rows and columns. **polars** gets even more powerful when you add in the ability to work with groups. And because groups tend to imply an aggregation or pooling of data, they go hand-in-glove with the application of a summary statistic.\n", "\n", "The diagram below gives a sense of how these operations can proceed together. Note that the 'split' operation is achieved through grouping, while apply produces summary statistics. At the end, you get a data frame with a new index (one entry per group) in what is shown as the 'combine' step.\n", "\n", @@ -981,9 +977,7 @@ "\n", "### Grouping and Aggregating\n", "\n", - "Let's take a look at creating a group using the `.groupby()` function followed by selecting a column and applying a summary statistic via an aggregation. Note that *aggregation*, via `.agg()`, always produces a new index because we have collapsed information down to the group-level (and the new index is made of those levels).\n", - "\n", - "The key point to remember is: use `.agg()` with `.groupby()` when you want your groups to become the new index." + "Let's take a look at creating a group using the `.group_by()` function, then followed by the `.agg()` function for aggregation, selecting a column and applying a summary statistic via an aggregation.\n" ] }, { @@ -993,7 +987,7 @@ "metadata": {}, "outputs": [], "source": [ - "(flights.groupby(\"month\")[[\"dep_delay\"]].mean())" + "flights.group_by(\"month\").agg(pl.col(\"dep_delay\").mean())" ] }, { @@ -1001,19 +995,9 @@ "id": "b003ea0d", "metadata": {}, "source": [ - "This now represents the mean departure delay by month. Notice that our index has changed! We now have month where we original had an index that was just the row number. The index plays an important role in grouping operations because it keeps track of the groups you have in the rest of your data frame.\n", + "This now represents the mean departure delay by month. The mechanics happenning here is that the DataFrame is grouped by each unique item in the _\"month\"_ column and then a mean summary statistic is derived on the _\"dep_delay\"_ column from each group.\n", "\n", - "Often, you might want to do multiple summary operations in one go. The most comprehensive syntax for this is via `.agg()`. We can reproduce what we did above using `.agg()`:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aaaca266", - "metadata": {}, - "outputs": [], - "source": [ - "(flights.groupby(\"month\")[[\"dep_delay\"]].agg(\"mean\"))" + "Other summary statistics can be derived from aggregations. Some common options are in the table below:\n" ] }, { @@ -1021,21 +1005,20 @@ "id": "520299c7", "metadata": {}, "source": [ - "where you pass in whatever aggregation you want. Some common options are in the table below:\n", - "\n", - "| Aggregation | Description |\n", - "| ----------- | ----------- |\n", - "| `count()` | Number of items |\n", - "| `first()`, `last()` | \tFirst and last item |\n", - "| `mean()`, `median()` |\tMean and median |\n", - "| `min()`, `max()` |\tMinimum and maximum |\n", - "| `std()`, `var()` |\tStandard deviation and variance |\n", - "| `mad()` |\tMean absolute deviation |\n", - "| `prod()` |\tProduct of all items |\n", - "| `sum()`\t| Sum of all items |\n", - "| `value_counts()` | Counts of unique values |\n", + "| Aggregation | Description |\n", + "| -------------------- | ------------------------------- |\n", + "| `count()` | Number of non-null items |\n", + "| `len()` | Number of all items |\n", + "| `first()`, `last()` | First and last item |\n", + "| `mean()`, `median()` | Mean and median |\n", + "| `min()`, `max()` | Minimum and maximum |\n", + "| `std()`, `var()` | Standard deviation and variance |\n", + "| `mad()` | Mean absolute deviation |\n", + "| `product()` | Product of all items |\n", + "| `sum()` | Sum of all items |\n", + "| `value_counts()` | Counts of unique values |\n", "\n", - "For doing multiple aggregations on multiple columns with new names for the output variables, the syntax becomes" + "For doing multiple aggregations on multiple columns with new names for the output variables, the syntax becomes\n" ] }, { @@ -1045,11 +1028,10 @@ "metadata": {}, "outputs": [], "source": [ - "(\n", - " flights.groupby([\"month\"]).agg(\n", - " mean_delay=(\"dep_delay\", \"mean\"),\n", - " count_flights=(\"dep_delay\", \"count\"),\n", - " )\n", + "# Multiple aggregations using polars' syntactic sugar (shorthand) for mean and count\n", + "flights.group_by(\"month\").agg(\n", + " mean_delay=pl.mean(\"dep_delay\"),\n", + " count_flights=pl.count(\"dep_delay\"),\n", ")" ] }, @@ -1058,7 +1040,7 @@ "id": "c331e813", "metadata": {}, "source": [ - "Means and counts can get you a surprisingly long way in data science!" + "Means and counts can get you a surprisingly long way in data science!\n" ] }, { @@ -1068,7 +1050,7 @@ "source": [ "### Grouping by multiple variables\n", "\n", - "This is as simple as passing `.groupby()` a list representing multiple columns instead of a string representing a single column." + "This is as simple as passing `.group_by()` a list or multiple strings representing columns instead of a single string representing a single column.\n" ] }, { @@ -1078,95 +1060,13 @@ "metadata": {}, "outputs": [], "source": [ - "month_year_delay = flights.groupby([\"month\", \"year\"]).agg(\n", - " mean_delay=(\"dep_delay\", \"mean\"),\n", - " count_flights=(\"dep_delay\", \"count\"),\n", + "month_year_delay = flights.group_by(\"month\", \"year\").agg(\n", + " mean_delay=pl.mean(\"dep_delay\"),\n", + " count_flights=pl.count(\"dep_delay\"),\n", ")\n", "month_year_delay" ] }, - { - "cell_type": "markdown", - "id": "22e89b7e", - "metadata": {}, - "source": [ - "You might have noticed that this time we have a multi-index (that is, an index with more than one column). That's because we asked for something with multiple groups, and the index tracks what's going on within each group: so we need more than one dimension of index to do this efficiently.\n", - "\n", - "If you ever want to go back to an index that is just the position, try `reset_index()`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "33712dc2", - "metadata": {}, - "outputs": [], - "source": [ - "month_year_delay.reset_index()" - ] - }, - { - "cell_type": "markdown", - "id": "96c8416a", - "metadata": {}, - "source": [ - "Perhaps you only want to remove one layer of the index though. This can be achieved by passing the position of the index you'd like to remove: for example, to only change the year index to a column, we would use: " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bcbef8f7", - "metadata": {}, - "outputs": [], - "source": [ - "month_year_delay.reset_index(1)" - ] - }, - { - "cell_type": "markdown", - "id": "8ab554b2", - "metadata": {}, - "source": [ - "Finally, you can do more complicated re-arrangements of the index with an operation called `unstack`, which pivots the chosen index variable to be a column variable instead (introducing a multi column level structure). It's usually best to avoid this." - ] - }, - { - "cell_type": "markdown", - "id": "b3e1ea60", - "metadata": {}, - "source": [ - "### Grouping and Transforming\n", - "\n", - "You may not always want to change the index to reflect new groups when performing computations at the group level.\n", - "\n", - "The key point to remember is: use `.transform()` with `.groupby()` when you want to perform computations on your groups but you want to go back to the original index.\n", - "\n", - "Let's say we wanted to express the arrival delay, `\"arr_del\"`, of each flight as a fraction of the worst arrival delay in each month." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f96fcf41", - "metadata": {}, - "outputs": [], - "source": [ - "flights[\"max_delay_month\"] = flights.groupby(\"month\")[\"arr_delay\"].transform(\"max\")\n", - "flights[\"delay_frac_of_max\"] = flights[\"arr_delay\"] / flights[\"max_delay_month\"]\n", - "flights[\n", - " [\"year\", \"month\", \"day\", \"arr_delay\", \"max_delay_month\", \"delay_frac_of_max\"]\n", - "].head()" - ] - }, - { - "cell_type": "markdown", - "id": "3880ece9", - "metadata": {}, - "source": [ - "Note that the first few entries of `\"max_delay_month\"` are all the same because the month is the same for those entries, but the delay fraction changes with each row." - ] - }, { "cell_type": "markdown", "id": "1954cb3b", @@ -1174,18 +1074,15 @@ "source": [ "### Groupby Exercises\n", "\n", - "1. Which carrier has the worst delays? Challenge: can you disentangle the effects of bad airports vs. bad carriers? Why/why not? (Hint: think about `flights.groupby([\"carrier\", \"dest\"]).count()`)\n", + "1. Which carrier has the worst delays? Challenge: can you disentangle the effects of bad airports vs. bad carriers? Why/why not? (Hint: think about `flights.group_by([\"carrier\", \"dest\"])`)\n", "\n", "2. Find the most delayed flight to each destination.\n", "\n", - "3. How do delays vary over the course of the day?" + "3. How do delays vary over the course of the day?\n" ] } ], "metadata": { - "interpreter": { - "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" - }, "jupytext": { "cell_metadata_filter": "-all", "encoding": "# -*- coding: utf-8 -*-", @@ -1193,7 +1090,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "python4ds", "language": "python", "name": "python3" }, @@ -1207,7 +1104,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/data-visualise.ipynb b/data-visualise.ipynb index 0335c48..8f3d721 100644 --- a/data-visualise.ipynb +++ b/data-visualise.ipynb @@ -17,7 +17,7 @@ "\n", "However, we'll get further faster by learning one system and applying it in many places—and the beauty of declarative plotting is that it covers lots of standard charts simply and well. **letsplot** implements the so-called **grammar of graphics**, a coherent declarative system for describing and building graphs.\n", "\n", - "We will start by creating a simple scatterplot and use that to introduce aesthetic mappings and geometric objects—the fundamental building blocks of **letsplot**. We will then walk you through visualising distributions of single variables as well as visualising relationships between two or more variables. We’ll finish off with saving your plots and troubleshooting tips. " + "We will start by creating a simple scatterplot and use that to introduce aesthetic mappings and geometric objects—the fundamental building blocks of **letsplot**. We will then walk you through visualising distributions of single variables as well as visualising relationships between two or more variables. We’ll finish off with saving your plots and troubleshooting tips.\n" ] }, { @@ -27,7 +27,7 @@ "source": [ "### Prerequisites\n", "\n", - "You will need to install the **letsplot** package for this chapter. To do this, open up the command line of your computer, type in `uv add lets-plot`, and hit enter." + "You will need to install the **letsplot** package for this chapter. To do this, open up the command line of your computer, type in `uv add lets-plot`, and hit enter.\n" ] }, { @@ -39,7 +39,7 @@ "The command line can be opened within Visual Studio Code and Codespaces by going to View -> Terminal.\n", ":::\n", "\n", - "Note that you only need to install a package once in each Python environment." + "Note that you only need to install a package once in each Python environment.\n" ] }, { @@ -47,9 +47,9 @@ "id": "e0ad70c8", "metadata": {}, "source": [ - "We'll also need to have the **pandas** package installed—this package, which we'll be seeing a lot of, is for data. You can similarly install it by running `uv add pandas` on the command line.\n", + "We'll also need to have the **polars** package installed—this package, which we'll be seeing a lot of, is for data. You can similarly install it by running `uv add polars` on the command line.\n", "\n", - "Finally, we'll also need some data (you can't science without data). We'll be using the Palmer penguins dataset. Unusually, this can also be installed as a package—normally you would load data from a file, but these data are so popular for tutorials they've found their way into an installable package. Run `uv add palmerpenguins` to get these data." + "Finally, we'll also need some data (you can't science without data). We'll be using the Palmer penguins dataset. Unusually, this can also be installed as a package—normally you would load data from a file, but these data are so popular for tutorials they've found their way into an installable package. Run `uv add palmerpenguins` to get these data.\n" ] }, { @@ -57,7 +57,7 @@ "id": "8852373a", "metadata": {}, "source": [ - "Our next task is to load these into our Python session, either in a Python notebook cell within a Jupyter Notebook, by writing it in a script that we then send to the interactive window, or by typing it directly into the interactive window and hitting shift and enter. Here's the code:" + "Our next task is to load these into our Python session, either in a Python notebook cell within a Marimo or Jupyter Notebook, by writing it in a script that we then send to the interactive window, or by typing it directly into the interactive window and hitting shift and enter. Here's the code:\n" ] }, { @@ -67,6 +67,7 @@ "metadata": {}, "outputs": [], "source": [ + "import polars as pl\n", "from lets_plot import *\n", "from palmerpenguins import load_penguins\n", "\n", @@ -78,7 +79,7 @@ "id": "4443f4dd", "metadata": {}, "source": [ - "These lines import parts of the **pandas** and **palmerpenguins** packages, then import all (`*`) of the functions of the **letsplot** package. The final line allows charts to display in HTML." + "These lines import parts of the **palmerpenguins** package, then import all (`*`) of the functions of the **letsplot** package. The final line allows charts to display in HTML.\n" ] }, { @@ -88,7 +89,7 @@ "source": [ "## First Steps\n", "\n", - "Do penguins with longer flippers weigh more or less than penguins with shorter flippers? You probably already have an answer, but try to make your answer precise. What does the relationship between flipper length and body mass look like? Is it positive? Negative? Linear? Nonlinear? Does the relationship vary by the species of the penguin? How about by the island where the penguin lives? Let’s create visualisations that we can use to answer these questions." + "Do penguins with longer flippers weigh more or less than penguins with shorter flippers? You probably already have an answer, but try to make your answer precise. What does the relationship between flipper length and body mass look like? Is it positive? Negative? Linear? Nonlinear? Does the relationship vary by the species of the penguin? How about by the island where the penguin lives? Let’s create visualisations that we can use to answer these questions.\n" ] }, { @@ -102,21 +103,21 @@ "\n", "To make the discussion easier, let's define some terms:\n", "\n", - "- A **variable** is a quantity, quality, or property that you can measure.\n", + "- A **variable** is a quantity, quality, or property that you can measure.\n", "\n", - "- A **value** is the state of a variable when you measure it.\n", - " The value of a variable may change from measurement to measurement.\n", + "- A **value** is the state of a variable when you measure it.\n", + " The value of a variable may change from measurement to measurement.\n", "\n", - "- An **observation** is a set of measurements made under similar conditions (you usually make all of the measurements in an observation at the same time and on the same object).\n", - " An observation will contain several values, each associated with a different variable.\n", - " We'll sometimes refer to an observation as a data point.\n", + "- An **observation** is a set of measurements made under similar conditions (you usually make all of the measurements in an observation at the same time and on the same object).\n", + " An observation will contain several values, each associated with a different variable.\n", + " We'll sometimes refer to an observation as a data point.\n", "\n", - "- **Tabular data** is a set of values, each associated with a variable and an observation.\n", - " Tabular data is *tidy* if each value is placed in its own \"cell\", each variable in its own column, and each observation in its own row.\n", + "- **Tabular data** is a set of values, each associated with a variable and an observation.\n", + " Tabular data is _tidy_ if each value is placed in its own \"cell\", each variable in its own column, and each observation in its own row.\n", "\n", "In this context, a variable refers to an attribute of all the penguins, and an observation refers to all the attributes of a single penguin.\n", "\n", - "Type the name of the data frame in the interactive window and Python will print a preview of its contents." + "Type the name of the data frame in the interactive window and Python will print a preview of its contents.\n" ] }, { @@ -126,7 +127,7 @@ "metadata": {}, "outputs": [], "source": [ - "penguins = load_penguins()\n", + "penguins = pl.from_pandas(load_penguins())\n", "penguins" ] }, @@ -135,7 +136,7 @@ "id": "cc310b4f", "metadata": {}, "source": [ - "For an alternative view, where you can see the first few observations of each variable, use `penguins.head()`." + "For an alternative view, where you can see the first few observations of each variable, use `penguins.head()`.\n" ] }, { @@ -171,7 +172,7 @@ "source": [ "### Ultimate Goal\n", "\n", - "Our ultimate goal in this chapter is to recreate the following visualisation displaying the relationship between flipper lengths and body masses of these penguins, taking into consideration the species of the penguin." + "Our ultimate goal in this chapter is to recreate the following visualisation displaying the relationship between flipper lengths and body masses of these penguins, taking into consideration the species of the penguin.\n" ] }, { @@ -241,7 +242,7 @@ "For example, bar charts use bar geoms (`geom_bar()`), line charts use line geoms (`geom_line()`), boxplots use boxplot geoms (`geom_boxplot()`), scatterplots use point geoms (`geom_point()`), and so on.\n", "\n", "The function `geom_point()` adds a layer of points to your plot, which creates a scatterplot.\n", - "**letsplot** comes with many geom functions that each adds a different type of layer to a plot." + "**letsplot** comes with many geom functions that each adds a different type of layer to a plot.\n" ] }, { @@ -266,7 +267,7 @@ "It doesn't yet match our \"ultimate goal\" plot, but using this plot we can start answering the question that motivated our exploration: \"What does the relationship between flipper length and body mass look like?\" The relationship appears to be positive (as flipper length increases, so does body mass), fairly linear (the points are clustered around a line instead of a curve), and moderately strong (there isn't too much scatter around such a line).\n", "Penguins with longer flippers are generally larger in terms of their body mass.\n", "\n", - "It's a good point to flag that although we have plotted everything in the `penguins` data frame, there were a couple of rows with undefined values—and of course these cannot be plotted." + "It's a good point to flag that although we have plotted everything in the `penguins` data frame, there were a couple of rows with undefined values—and of course these cannot be plotted.\n" ] }, { @@ -285,7 +286,7 @@ "If you guessed \"in the aesthetic mapping, inside of `aes()`\", you're already getting the hang of creating data visualisations with **letsplot**!\n", "And if not, don't worry.\n", "\n", - "Throughout the book you will make many more plots and have many more opportunities to check your intuition as you make them." + "Throughout the book you will make many more plots and have many more opportunities to check your intuition as you make them.\n" ] }, { @@ -319,7 +320,7 @@ "\n", "Since this is a new geometric object representing our data, we will add a new geom as a layer on top of our point geom: `geom_smooth()`.\n", "\n", - "And we will specify that we want to draw the line of best fit based on a `l`inear `m`odel with `method = \"lm\"`." + "And we will specify that we want to draw the line of best fit based on a `l`inear `m`odel with `method = \"lm\"`.\n" ] }, { @@ -346,9 +347,9 @@ "source": [ "We have successfully added lines, but this plot doesn't look like the plot from earlier as that only had one line for the entire dataset as opposed to separate lines for each of the penguin species.\n", "\n", - "When aesthetic mappings are defined in `ggplot()`, at the *global* level, they're passed down to each of the subsequent geom layers of the plot.\n", + "When aesthetic mappings are defined in `ggplot()`, at the _global_ level, they're passed down to each of the subsequent geom layers of the plot.\n", "\n", - "However, each geom function in **letplot** can also take a `mapping` argument, which allows for aesthetic mappings at the *local* level that are added to those inherited from the global level.\n", + "However, each geom function in **letplot** can also take a `mapping` argument, which allows for aesthetic mappings at the _local_ level that are added to those inherited from the global level.\n", "\n", "Since we want points to be colored based on species but don't want the lines to be separated out for them, we should specify `color = species` for `geom_point()` only: therefore we take it out of the global `aes()` and just add it to `geom_point()`.\n" ] @@ -376,7 +377,7 @@ "\n", "We still need to use different shapes for each species of penguins and improve labels.\n", "\n", - "It's generally not a good idea to represent information using only colors on a plot, as people perceive colors differently due to color blindness or other color vision differences. Therefore, in addition to color, we can also map `species` to the `shape` aesthetic." + "It's generally not a good idea to represent information using only colors on a plot, as people perceive colors differently due to color blindness or other color vision differences. Therefore, in addition to color, we can also map `species` to the `shape` aesthetic.\n" ] }, { @@ -400,7 +401,7 @@ "source": [ "Note that the legend is automatically updated to reflect the different shapes of the points as well.\n", "\n", - "And finally, we can improve the labels of our plot using the `labs()` function in a new layer. Some of the arguments to `labs()` might be self explanatory: `title` adds a title and `subtitle` adds a subtitle to the plot. Other arguments match the aesthetic mappings, `x` is the x-axis label, `y` is the y-axis label, and `color` and `shape` define the label for the legend." + "And finally, we can improve the labels of our plot using the `labs()` function in a new layer. Some of the arguments to `labs()` might be self explanatory: `title` adds a title and `subtitle` adds a subtitle to the plot. Other arguments match the aesthetic mappings, `x` is the x-axis label, `y` is the y-axis label, and `color` and `shape` define the label for the legend.\n" ] }, { @@ -430,7 +431,7 @@ "id": "cdc33b33", "metadata": {}, "source": [ - "We finally have a plot that perfectly matches our \"ultimate goal\"!" + "We finally have a plot that perfectly matches our \"ultimate goal\"!\n" ] }, { @@ -456,7 +457,7 @@ "5. Why does the following give an error and how would you fix it?\n", "\n", " ```python\n", - " (ggplot(data = penguins) + \n", + " (ggplot(data = penguins) +\n", " geom_point())\n", " ```\n", "\n", @@ -464,7 +465,7 @@ "\n", "7. Recreate the following visualisation.\n", " What aesthetic should `bill_depth_mm` be mapped to?\n", - " And should it be mapped at the global level or at the geom level?" + " And should it be mapped at the global level or at the geom level?\n" ] }, { @@ -490,7 +491,6 @@ "id": "986fdc29", "metadata": {}, "source": [ - "\n", "8. Run this code in your head and predict what the output will look like.\n", " Then, run the code in Python and check your predictions.\n", "\n", @@ -518,6 +518,7 @@ " geom_smooth()\n", " )\n", " ```\n", + "\n", " ```python\n", " (ggplot() +\n", " geom_point(\n", @@ -529,7 +530,7 @@ " mapping = aes(x = \"flipper_length_mm\", y = \"body_mass_g\")\n", " )\n", " )\n", - " ```" + " ```\n" ] }, { @@ -549,7 +550,7 @@ " mapping = aes(x = \"flipper_length_mm\", y = \"body_mass_g\")\n", ") +\n", " geom_point())\n", - "```" + "```\n" ] }, { @@ -565,10 +566,10 @@ "\n", "```python\n", "(\n", - " ggplot(penguins, aes(x = \"flipper_length_mm\", y = \"body_mass_g\")) + \n", + " ggplot(penguins, aes(x = \"flipper_length_mm\", y = \"body_mass_g\")) +\n", " geom_point()\n", ")\n", - "```" + "```\n" ] }, { @@ -602,9 +603,9 @@ "id": "699f42eb", "metadata": {}, "source": [ - "You may have seen earlier that the *data type* of the `\"species\"` column is string. Ideally, we want it to be categorical, so that there's no confusion about the fact that we're dealing with a finite number of mutually exclusive groups here. Another advantage is that it allows plotting tools to realise what kind of data it is working with.\n", + "You may have seen earlier that the _data type_ of the `\"species\"` column is string. Ideally, we want it to be categorical, so that there's no confusion about the fact that we're dealing with a finite number of mutually exclusive groups here. Another advantage is that it allows plotting tools to realise what kind of data it is working with.\n", "\n", - "We can transform the variable to a categorical variable using **pandas** like so:" + "We can transform the variable to a categorical variable using **polars** like so:\n" ] }, { @@ -614,7 +615,7 @@ "metadata": {}, "outputs": [], "source": [ - "penguins[\"species\"] = penguins[\"species\"].astype(\"category\")\n", + "penguins = penguins.cast({\"species\": pl.Categorical})\n", "penguins.head()" ] }, @@ -623,7 +624,7 @@ "id": "06d834a5", "metadata": {}, "source": [ - "You will learn more about categorical variables later in the book." + "You will learn more about categorical variables later in the book.\n" ] }, { @@ -631,12 +632,11 @@ "id": "f9ca3124", "metadata": {}, "source": [ - "\n", "### A numerical variable\n", "\n", "A variable is **numerical** (or quantitative) if it can take on a wide range of numerical values, and it is sensible to add, subtract, or take averages with those values. Numerical variables can be continuous or discrete.\n", "\n", - "One commonly used visualisation for distributions of continuous variables is a histogram." + "One commonly used visualisation for distributions of continuous variables is a histogram.\n" ] }, { @@ -661,7 +661,7 @@ "You should always explore a variety of binwidths when working with histograms, as different binwidths can reveal different patterns.\n", "In the plots below a binwidth of 20 is too narrow, resulting in too many bars, making it difficult to determine the shape of the distribution.\n", "Similarly, a binwidth of 2,000 is too high, resulting in all data being binned into only three bars, and also making it difficult to determine the shape of the distribution.\n", - "A binwidth of 200 provides a sensible balance, but you should always look at your data a few different ways, especially with histograms as they can be misleading." + "A binwidth of 200 provides a sensible balance, but you should always look at your data a few different ways, especially with histograms as they can be misleading.\n" ] }, { @@ -710,7 +710,7 @@ " geom_bar(fill = \"red\"))\n", " ```\n", "\n", - "3. What does the `bins` argument in `geom_histogram()` do?" + "3. What does the `bins` argument in `geom_histogram()` do?\n" ] }, { @@ -722,7 +722,7 @@ "\n", "To visualise a relationship we need to have at least two variables mapped to aesthetics of a plot—though you should remember that correlation is not causation, and causation is not correlation!\n", "\n", - "In the following sections you will learn about commonly used plots for visualising relationships between two or more variables and the geoms used for creating them." + "In the following sections you will learn about commonly used plots for visualising relationships between two or more variables and the geoms used for creating them.\n" ] }, { @@ -738,17 +738,16 @@ "\n", "It is also useful for identifying potential outliers. Each boxplot consists of:\n", "\n", - "- A box that indicates the range of the middle half of the data, a distance known as the interquartile range (IQR), stretching from the 25th percentile of the distribution to the 75th percentile.\n", - " In the middle of the box is a line that displays the median, i.e. 50th percentile, of the distribution.\n", - " These three lines give you a sense of the spread of the distribution and whether or not the distribution is symmetric about the median or skewed to one side.\n", - "\n", - "- Visual points that display observations that fall more than 1.5 times the IQR from either edge of the box.\n", - " These outlying points are unusual so are plotted individually.\n", + "- A box that indicates the range of the middle half of the data, a distance known as the interquartile range (IQR), stretching from the 25th percentile of the distribution to the 75th percentile.\n", + " In the middle of the box is a line that displays the median, i.e. 50th percentile, of the distribution.\n", + " These three lines give you a sense of the spread of the distribution and whether or not the distribution is symmetric about the median or skewed to one side.\n", "\n", - "- A line (or whisker) that extends from each end of the box and goes to the farthest non-outlier point in the distribution.\n", + "- Visual points that display observations that fall more than 1.5 times the IQR from either edge of the box.\n", + " These outlying points are unusual so are plotted individually.\n", "\n", + "- A line (or whisker) that extends from each end of the box and goes to the farthest non-outlier point in the distribution.\n", "\n", - "Let's take a look at the distribution of body mass by species using `geom_boxplot()`:" + "Let's take a look at the distribution of body mass by species using `geom_boxplot()`:\n" ] }, { @@ -766,7 +765,7 @@ "id": "97b24caa", "metadata": {}, "source": [ - "Alternatively, we can make probability density plots with `geom_density()`." + "Alternatively, we can make probability density plots with `geom_density()`.\n" ] }, { @@ -788,7 +787,7 @@ "\n", "Additionally, we can map `species` to both `color` and `fill` aesthetics and use the `alpha` aesthetic to add transparency to the filled density curves.\n", "This aesthetic takes values between 0 (completely transparent) and 1 (completely opaque).\n", - "In the following plot it's *set* to 0.5." + "In the following plot it's _set_ to 0.5.\n" ] }, { @@ -811,8 +810,8 @@ "source": [ "Note the terminology we have used here:\n", "\n", - "- We *map* variables to aesthetics if we want the visual attribute represented by that aesthetic to vary based on the values of that variable.\n", - "- Otherwise, we *set* the value of an aesthetic.\n" + "- We _map_ variables to aesthetics if we want the visual attribute represented by that aesthetic to vary based on the values of that variable.\n", + "- Otherwise, we _set_ the value of an aesthetic.\n" ] }, { @@ -829,7 +828,7 @@ "The first plot shows the frequencies of each species of penguins on each island.\n", "The plot of frequencies show that there are equal numbers of Adelies on each island.\n", "\n", - "But we don't have a good sense of the percentage balance within each island." + "But we don't have a good sense of the percentage balance within each island.\n" ] }, { @@ -867,7 +866,7 @@ "id": "cc83c3db", "metadata": {}, "source": [ - "In creating these bar charts, we map the variable that will be separated into bars to the `x` aesthetic, and the variable that will change the colors inside the bars to the `fill` aesthetic." + "In creating these bar charts, we map the variable that will be separated into bars to the `x` aesthetic, and the variable that will change the colors inside the bars to the `fill` aesthetic.\n" ] }, { @@ -927,7 +926,7 @@ "\n", "To facet your plot by a single variable, use `facet_wrap()`.\n", "\n", - "The first argument of `facet_wrap()` tells the function what variable to have in successive charts. The variable that you pass to `facet_wrap()` should be categorical." + "The first argument of `facet_wrap()` tells the function what variable to have in successive charts. The variable that you pass to `facet_wrap()` should be categorical.\n" ] }, { @@ -949,7 +948,7 @@ "id": "ee5a3eed", "metadata": {}, "source": [ - "You will learn about many other geoms for visualising distributions of variables and relationships between them in later chapters." + "You will learn about many other geoms for visualising distributions of variables and relationships between them in later chapters.\n" ] }, { @@ -971,7 +970,7 @@ " ggplot(\n", " data = penguins,\n", " mapping = aes(\n", - " x = \"bill_length_mm\", y = \"bill_depth_mm\", \n", + " x = \"bill_length_mm\", y = \"bill_depth_mm\",\n", " color = \"species\", shape = \"species\"\n", " )\n", " ) +\n", @@ -1023,9 +1022,9 @@ "source": [ "This saved the figure to disk at the location shown—by default it's in a subdirectory called \"lets-plot-images\".\n", "\n", - "We used the file format \"svg\". There are lots of output options to choose from to save your file to. Remember that, for graphics, *vector formats* are generally better than *raster formats*. In practice, this means saving plots in svg or pdf formats over jpg or png file formats. The svg format works in a lot of contexts (including Microsoft Word) and is a good default. To choose between formats, just supply the file extension and the file type will change automatically, eg \"chart.svg\" for svg or \"chart.png\" for png. You can also save figures in HTML format.\n", + "We used the file format \"svg\". There are lots of output options to choose from to save your file to. Remember that, for graphics, _vector formats_ are generally better than _raster formats_. In practice, this means saving plots in svg or pdf formats over jpg or png file formats. The svg format works in a lot of contexts (including Microsoft Word) and is a good default. To choose between formats, just supply the file extension and the file type will change automatically, eg \"chart.svg\" for svg or \"chart.png\" for png. You can also save figures in HTML format.\n", "\n", - "If you're using a raster format then you'll need to specify how big the figure is via the *scale* keyword argument." + "If you're using a raster format then you'll need to specify how big the figure is via the _scale_ keyword argument.\n" ] }, { @@ -1051,7 +1050,7 @@ "source": [ "### Exercises\n", "\n", - "1. Save the figure above as a PNG. Try varying the scale." + "1. Save the figure above as a PNG. Try varying the scale.\n" ] }, { @@ -1066,13 +1065,12 @@ "We have all been writing Python code for years, but every day we still write code that doesn't work on the first try!\n", "\n", "Start by carefully comparing the code that you're running to the code in the book: A misplaced character can make all the difference!\n", - "Make sure that every `(` is matched with a `)` and every `\"` is paired with another `\"`. In Visual Studio Code, you can get extensions that colour match brackets so you can easily see if you closed them or not.\n", + "Make sure that every `(` is matched with a `)` and every `\"` is paired with another `\"`. In Visual Studio Code, you can get extensions that colour match brackets so you can easily see if you closed them or not.\n", "\n", "Sometimes you'll run the code and nothing happens.\n", "\n", "For those coming from the R statistical programming language, you may be concerned about getting your `+` in the wrong place. Have no fear, however, as in the syntax for **letsplot** the `+` can go at the start or the end of the line.\n", "\n", - "\n", "If you're still stuck, try the help.\n", "You can get help about any Python function by running `help(function_name)` in the interactive window.\n", "Don't worry if the help doesn't seem that helpful - instead skip down to the examples and look for code that matches what you're trying to do.\n", @@ -1095,7 +1093,7 @@ "We'll use visualisations again and again throughout this book, introducing new techniques as we need them as well as do a deeper dive into creating visualisations with **letsplot** in subsequent chapters.\n", "\n", "With the basics of visualisation under your belt, in the next chapter we're going to switch gears a little and give you some practical workflow advice.\n", - "We intersperse workflow advice with data science tools throughout this part of the book because it'll help you stay organised as you write more Python code." + "We intersperse workflow advice with data science tools throughout this part of the book because it'll help you stay organised as you write more Python code.\n" ] } ], @@ -1107,7 +1105,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": ".venv", + "display_name": "python4ds", "language": "python", "name": "python3" }, @@ -1121,7 +1119,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/data/bake_sale.xlsx b/data/bake_sale.xlsx index cf475a2..5de6f9a 100644 Binary files a/data/bake_sale.xlsx and b/data/bake_sale.xlsx differ diff --git a/databases.ipynb b/databases.ipynb index f14914c..b183959 100644 --- a/databases.ipynb +++ b/databases.ipynb @@ -17,7 +17,7 @@ "\n", "### Prerequisites\n", "\n", - "You will need the **pandas**, **SQLModel**, and **ibis** packages for this chapter. You probably already have **pandas** installed; to install **SQLModel** and **ibis** respectively run `uv add sqlmodel` and `uv add ibis-framework` on your computer's command line. First, let's bring in some general packages and turn off verbose warnings." + "You will need the **polars**, **SQLModel**, and **ibis** packages for this chapter. You probably already have **polars** installed; to install **SQLModel** and **ibis** respectively run `uv add sqlmodel` and `uv add ibis-framework` on your computer's command line. First, let's bring in some general packages and turn off verbose warnings." ] }, { @@ -39,10 +39,9 @@ "metadata": {}, "source": [ "## Database Basics\n", - "\n", - "At the simplest level, you can think about a database as a collection of data frames, called **tables** in database terminology.\n", - "Like a **pandas** data frame, a database table is a collection of named columns, where every value in the column is the same type.\n", - "There are three high level differences between data frames and database tables:\n", + "At the simplest level, you can think about a database as a collection of data frames, called **tables** in database terminology. \n", + "Like a **Polars** DataFrame, a database table is a collection of named columns, where every value in a column shares the same data type. \n", + "There are three high-level differences between data frames and database tables:\n", "\n", "- Database tables are stored on disk (ie on file) and can be arbitrarily large.\n", " Data frames are stored in memory, and are fundamentally limited (although that limit is still big enough for many problems). You can think about the difference between on disk and in memory as being like the difference between long-term and short-term memory (and you have much more limited capacity in the latter).\n", @@ -68,7 +67,7 @@ "\n", "- You'll always use a database interface that provides a connection to the database, for example Python's built-in **sqlite** package\n", "\n", - "- You'll also use a package that pushes and/or pulls data to/from the database, for example **pandas**\n", + "- You'll also use a package that pushes and/or pulls data to/from the database, for example **polars**\n", "\n", "The precise details of the connection varies a lot from DBMS to DBMS so unfortunately we can't cover all the details here. The initial setup will often take a little fiddling (and maybe some research) to get right, but you'll generally only need to do it once. We'll do the best we can to cover some basics here.\n", "\n", @@ -112,7 +111,7 @@ "id": "2992b718", "metadata": {}, "source": [ - "Note that the output here is in the form a Python object called a tuple. If we wanted to put this into a **pandas** data frame, we can just pass it straight in:" + "Note that the output here is in the form of a Python object called a tuple. If we want to convert this into a **Polars** DataFrame, we can pass it to `pl.DataFrame()`. When working with tuples, you may need to provide column names using the **schema** argument or specify **orient=\"row\"** so Polars correctly interprets the structure." ] }, { @@ -122,9 +121,11 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import polars as pl\n", + "\n", + "df = pl.DataFrame(rows, orient=\"row\")\n", "\n", - "pd.DataFrame(rows)" + "df" ] }, { @@ -316,9 +317,9 @@ "source": [ "### Joins\n", "\n", - "If you're familiar with joins in **pandas**, SQL joins are very similar. Let's see if we can join the 'album' and 'track' tables to find the *name* of the albums in the above query.\n", + "If you’re familiar with joins in **polars**, SQL joins are very similar. Let’s see if we can join the 'album' and 'track' tables to find the *name* of the albums in the above query.\n", "\n", - "Note that as soon as we have the *same* column names in more than one table, we need to specify the table we are referring to when we use that column name. There are different options for joins (eg `INNER`, `LEFT`) that you can find out more about [here](https://en.wikipedia.org/wiki/Join_(SQL)).\n" + "In polars, you use the `df.join()` method, which defaults to an \"inner\" join. Note that if you have the same column names in both tables, Polars will often append a suffix (like _right) to the duplicate names to keep them distinct, unless you specify otherwise. There are different options for joins (eg `INNER`, `LEFT`) that you can find out more about [here](https://en.wikipedia.org/wiki/Join_(SQL)).\n" ] }, { @@ -403,9 +404,9 @@ "id": "495f97e5", "metadata": {}, "source": [ - "## SQL with **pandas**\n", + "## SQL with **polars**\n", "\n", - "**pandas** is well-equipped for working with SQL. We can simply push the query we just created straight through using its `read_sql()` function—but bear in mind we need to pass in the connection we created to the database too:" + "**polars** is well-equipped for working with SQL. We can simply push the query we just created straight through using its `read_database()` function—but bear in mind we need to pass in the connection we created to the database too:" ] }, { @@ -415,7 +416,10 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_sql(sql_join, con)" + "df = pl.read_database(\n", + " query=sql_join, # your SQL query (string)\n", + " connection=con, # your connection object (SQLAlchemy, psycopg2 cursor, etc.)\n", + ")" ] }, { @@ -435,7 +439,7 @@ "source": [ "## SQL with **ibis**\n", "\n", - "It's not exactly satisfactory to have to write out your SQL queries in text. What if we could create commands directly from **pandas** commands? You can't *quite* do that, but there's a package that gets you pretty close and it's called [**ibis**](https://ibis-project.org/). **ibis** is particularly useful when you are reading from a database and want to query it just like you would a **pandas** data frame.\n", + "It's not exactly satisfactory to have to write out your SQL queries in text. What if we could create commands directly from **polars** commands? You can't *quite* do that, but there's a package that gets you pretty close and it's called [**ibis**](https://ibis-project.org/). **ibis** is particularly useful when you are reading from a database and want to query it just like you would a **polars** data frame.\n", "\n", "**Ibis** can connect to local databases (eg a SQLite database), server-based databases (eg Postgres), or cloud-based databased (eg Google's BigQuery). The syntax to make a connection is, for example, `ibis.bigquery.connect`.\n", "\n", @@ -462,7 +466,7 @@ "id": "6dcd7d71", "metadata": {}, "source": [ - "Okay, now let's reproduce the following query: \"SELECT albumid, AVG(milliseconds)/1e3/60 FROM track GROUP BY albumid ORDER BY AVG(milliseconds) ASC LIMIT 5;\". We'll use a groupby, a mutate (which you can think of like **pandas**' assign statement), a sort, and then `limit()` to only show the first five entries." + "Okay, now let's reproduce the following query: \"SELECT albumid, AVG(milliseconds)/1e3/60 FROM track GROUP BY albumid ORDER BY AVG(milliseconds) ASC LIMIT 5;\". We'll use a group_by, a mutate (which you can think of like **polars** assign statement), a sort, and then `limit()` to only show the first five entries." ] }, { @@ -793,7 +797,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/dataframe_illustration.svg b/dataframe_illustration.svg new file mode 100644 index 0000000..2c35021 --- /dev/null +++ b/dataframe_illustration.svg @@ -0,0 +1,233 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dates-and-times.ipynb b/dates-and-times.ipynb index 4fe0c68..47b53ad 100644 --- a/dates-and-times.ipynb +++ b/dates-and-times.ipynb @@ -1103,7 +1103,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/exploratory-data-analysis.ipynb b/exploratory-data-analysis.ipynb index 2bf323b..071a96b 100644 --- a/exploratory-data-analysis.ipynb +++ b/exploratory-data-analysis.ipynb @@ -1119,7 +1119,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/functions.ipynb b/functions.ipynb index 7171d21..5ed6b2a 100644 --- a/functions.ipynb +++ b/functions.ipynb @@ -31,7 +31,7 @@ "\n", "### Prerequisites\n", "\n", - "You will need the **pandas** and **numpy** packages for this chapter." + "You will need the **polars** and **numpy** packages for this chapter.\n" ] }, { @@ -43,7 +43,7 @@ "\n", "A function has inputs, it performs its function, and it returns any outputs. Functions begin with a def keyword for ‘define a function’. It then has a name, followed by brackets, (), which may contain function arguments and function keyword arguments. This is followed by a colon. The body of the function is then indented relative to the left-most text. Function arguments are defined in brackets following the name, with different inputs separated by commas. Any outputs are given with the return keyword, again with different variables separated by commas.\n", "\n", - "Let's see a very simple example of a function with a single *argument* (or arg):" + "Let's see a very simple example of a function with a single _argument_ (or arg):\n" ] }, { @@ -68,7 +68,7 @@ "id": "08c06c6a", "metadata": {}, "source": [ - "One powerful feature of functions is that we can define defaults for the input arguments. These are called *keyword arguments* (or kwargs). Let's see that in action by defining a default value for `name`, along with multiple outputs--a hello message and a score." + "One powerful feature of functions is that we can define defaults for the input arguments. These are called _keyword arguments_ (or kwargs). Let's see that in action by defining a default value for `name`, along with multiple outputs--a hello message and a score.\n" ] }, { @@ -107,17 +107,16 @@ "source": [ "::: {.callout-note title=\"Arguments and keyword arguments\"}\n", "\n", - "*arguments* are the variables that functions *always* need, so `a` and `b` in `def add(a, b): return a + b`. The function won't work without them! Function arguments are sometimes referred to as *args*.\n", + "_arguments_ are the variables that functions _always_ need, so `a` and `b` in `def add(a, b): return a + b`. The function won't work without them! Function arguments are sometimes referred to as _args_.\n", "\n", - "*Keyword arguments* are the variables that are optional for functions, so `c` in `def add(a, b, c=5): return a + b - c`. If you do not provide a value for `c` when calling the function, it will automatically revert to `c=5`. Keyword arguments are sometimes referred to as *kwargs*.\n", + "_Keyword arguments_ are the variables that are optional for functions, so `c` in `def add(a, b, c=5): return a + b - c`. If you do not provide a value for `c` when calling the function, it will automatically revert to `c=5`. Keyword arguments are sometimes referred to as _kwargs_.\n", ":::\n", "\n", - "\n", "::: {.callout-tip title=\"Exercise\"}\n", "What is the return type of a function with multiple return values separated by commas following the `return` statement?\n", ":::\n", "\n", - "In that last example, you'll notice that we added some text to the function. This is a doc-string, or documentation string. It's there to help users (and, most likely, future you) to understand what the function does. Let's see how this works in action by calling `help()` on the `score_message` function:" + "In that last example, you'll notice that we added some text to the function. This is a doc-string, or documentation string. It's there to help users (and, most likely, future you) to understand what the function does. Let's see how this works in action by calling `help()` on the `score_message` function:\n" ] }, { @@ -143,7 +142,7 @@ "Write a doc-string for your function and call `help` on it.\n", ":::\n", "\n", - "To learn more about args and kwargs, check out these [short video tutorials](https://calmcode.io/args-kwargs/introduction.html)." + "To learn more about args and kwargs, check out these [short video tutorials](https://calmcode.io/args-kwargs/introduction.html).\n" ] }, { @@ -155,25 +154,39 @@ "\n", "You should consider writing a function whenever you've copied and pasted a block of code more than twice (i.e. you now have three copies of the same code).\n", "For example, take a look at this code.\n", - "What does it do?" + "What does it do?\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "7a744085", + "id": "43d10e40", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", - "import pandas as pd\n", - "\n", - "df = pd.DataFrame(np.random.normal(size=(10, 4)), columns=[\"a\", \"b\", \"c\", \"d\"])\n", - "\n", - "df[\"a\"] = (df[\"a\"] - df[\"a\"].min()) / (df[\"a\"].max() - df[\"a\"].min())\n", - "df[\"b\"] = (df[\"b\"] - df[\"b\"].min()) / (df[\"b\"].max() - df[\"a\"].min())\n", - "df[\"c\"] = (df[\"c\"] - df[\"c\"].min()) / (df[\"c\"].max() - df[\"c\"].min())\n", - "df[\"d\"] = (df[\"d\"] - df[\"d\"].min()) / (df[\"d\"].max() - df[\"d\"].min())" + "import polars as pl\n", + "\n", + "df = pl.DataFrame(np.random.normal(size=(10, 4)), schema=[\"a\", \"b\", \"c\", \"d\"])\n", + "\n", + "df = df.with_columns(\n", + " [\n", + " (\n", + " (pl.col(\"a\") - pl.col(\"a\").min()) / (pl.col(\"a\").max() - pl.col(\"a\").min())\n", + " ).alias(\"a\"),\n", + " (\n", + " (pl.col(\"b\") - pl.col(\"b\").min()) / (pl.col(\"b\").max() - pl.col(\"a\").min())\n", + " ).alias(\"b\"),\n", + " (\n", + " (pl.col(\"c\") - pl.col(\"c\").min()) / (pl.col(\"c\").max() - pl.col(\"c\").min())\n", + " ).alias(\"c\"),\n", + " (\n", + " (pl.col(\"d\") - pl.col(\"d\").min()) / (pl.col(\"d\").max() - pl.col(\"d\").min())\n", + " ).alias(\"d\"),\n", + " ]\n", + ")\n", + "\n", + "df" ] }, { @@ -182,19 +195,19 @@ "metadata": {}, "source": [ "You might be able to puzzle out that this rescales each column to have a range from 0 to 1.\n", - "But did you spot the mistake? There was an error when copying-and-pasting the code for `df[\"b\"]`: someone forgot to change an `a` to a `b`.\n", + "But did you spot the mistake? There was an error when copying-and-pasting the code for `\"b\"`: someone forgot to change an `a` to a `b`.\n", "Extracting repeated code out into a function is a good idea because it prevents you from making this type of mistake.\n", "\n", "To write a function you need to first analyse the code.\n", "How many inputs does it have?\n", "\n", "```python\n", - "df[\"a\"] - df[\"a\"].min() / (df[\"a\"].max() - df[\"a\"].min())\n", + "pl.col(\"a\") - pl.col(\"a\").min() / (pl.col(\"a\").max() - pl.col(\"a\").min())\n", "```\n", "\n", - "This code only has one input: `df[\"a\"]`. To make the inputs more clear, it's a good idea to rewrite the code using temporary variables with general names. Here this code only requires a single numeric vector, so we'll call it `x` and put it into a function.\n", + "This code only has one input: `pl.col(\"a\")`. To make the inputs more clear, it’s a good idea to rewrite the code using temporary variables with general names. In Polars, this code requires a column expression, so we’ll call it `x` and put it into a function.\n", "\n", - "Functions begin with a def keyword for ‘define a function’. It then has a name, followed by brackets, (), which may contain function arguments and function keyword arguments. This is followed by a colon. The body of the function is then indented relative to the left-most text. Function arguments are defined in brackets following the name, with different inputs separated by commas. Any outputs are given with the return keyword, again with different variables separated by commas.\n", + "Functions begin with a def keyword for 'define a function'. It then has a name, followed by brackets, (), which contain the function argument (our expression `x`). This is followed by a colon. The body of the function is then indented relative to the left-most text. Any outputs—which in this case will be a transformed expression—are given with the return keyword.\n", "\n", "So, in Python, functions have the form:\n", "\n", @@ -204,7 +217,7 @@ " \n", "```\n", "\n", - "So here it would be:" + "So here it would be:\n" ] }, { @@ -215,7 +228,7 @@ "```python\n", "def rescale(x):\n", " return (x - x.min()) / (x.max() - x.min())\n", - "```" + "```\n" ] }, { @@ -223,7 +236,7 @@ "id": "22f6afb7", "metadata": {}, "source": [ - "There is still some duplication in this code. We're computing the minimum of the data twice, so it makes sense to do it in once:" + "There is still some duplication in this code. We're computing the minimum of the data twice, so it makes sense to do it in once:\n" ] }, { @@ -245,40 +258,39 @@ "source": [ "Pulling out intermediate calculations into named variables is a good practice because it makes it more clear what the code is doing.\n", "\n", - "\n", "There are three key steps to creating a new function:\n", "\n", - "1. You need to pick a *name* for the function. Here we've used `rescale` because this function rescales a vector to lie between 0 and 1.\n", + "1. You need to pick a _name_ for the function. Here we've used `rescale` because this function rescales a vector to lie between 0 and 1.\n", "\n", - "1. You list the inputs, or *arguments*, to the function inside `function`.\n", + "1. You list the inputs, or _arguments_, to the function inside `function`.\n", " Here we have just one argument.\n", - " If we had more the call would look like `function(x, y, z)`. (We might also have a named *keyword argument* such as `data=` following the *arguments*.)\n", + " If we had more the call would look like `function(x, y, z)`. (We might also have a named _keyword argument_ such as `data=` following the _arguments_.)\n", "\n", - "3. You place the code you have developed in the *body* of the function, a block that immediately follows `function(...):`.\n", + "1. You place the code you have developed in the _body_ of the function, a block that immediately follows `function(...):`.\n", "\n", "Note the overall process: we only made the function after we'd figured out how to make it work with a simple input. It's easier to start with working code and turn it into a function; it's harder to create a function and then try to make it work.\n", "\n", - "At this point it's a good idea to check your function with a few different inputs:" + "At this point it's a good idea to check your function with a few different inputs:\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "b72eafca", + "id": "74834d90", "metadata": {}, "outputs": [], "source": [ - "rescale(pd.Series([-10, 0, 10]))" + "rescale(pl.Series([-10, 0, 10]))" ] }, { "cell_type": "code", "execution_count": null, - "id": "edd018c7", + "id": "8a0e643e", "metadata": {}, "outputs": [], "source": [ - "rescale(pd.Series([1, 2, 3, np.nan, 5]))" + "rescale(pl.Series([1, 2, 3, None, 5]))" ] }, { @@ -289,7 +301,7 @@ "As you write more and more functions you'll eventually want to convert these informal, interactive tests into formal, automated tests.\n", "That process is called unit testing. Unfortunately, it's beyond the scope of this book.\n", "\n", - "We can simplify the original example now that we have a function:" + "We can simplify the original example now that we have a function:\n" ] }, { @@ -299,10 +311,14 @@ "metadata": {}, "outputs": [], "source": [ - "df[\"a\"] = rescale(df[\"a\"])\n", - "df[\"b\"] = rescale(df[\"b\"])\n", - "df[\"c\"] = rescale(df[\"c\"])\n", - "df[\"d\"] = rescale(df[\"d\"])" + "df = df.with_columns(\n", + " [\n", + " rescale(pl.col(\"a\")).alias(\"a\"),\n", + " rescale(pl.col(\"b\")).alias(\"b\"),\n", + " rescale(pl.col(\"c\")).alias(\"c\"),\n", + " rescale(pl.col(\"d\")).alias(\"d\"),\n", + " ]\n", + ")" ] }, { @@ -313,17 +329,17 @@ "Compared to the original, this code is easier to understand and we've eliminated one class of copy-and-paste errors. There is still quite a bit of duplication since we're doing the same thing to multiple columns; we can actually remove this too, but we'll cover how to do that later in the book.\n", "\n", "Another advantage of functions is that if our requirements change, we only need to make the change in one place.\n", - "For example, we might discover that some of our variables include infinite values, and `rescale()` (effectively) fails:" + "For example, we might discover that some of our variables include infinite values, and `rescale()` (effectively) fails:\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "f503c36b", + "id": "f8b80894", "metadata": {}, "outputs": [], "source": [ - "rescale(pd.Series([1, 2, 3, np.inf, 5]))" + "rescale(pl.Series([1, 2, 3, float(\"inf\"), 5], strict=False))" ] }, { @@ -331,23 +347,23 @@ "id": "f29a050e", "metadata": {}, "source": [ - "Because we've extracted the code into a function, we only need to make the fix in one place:" + "Because we've extracted the code into a function, we only need to make the fix in one place:\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "67c861b6", + "id": "cd63ddea", "metadata": {}, "outputs": [], "source": [ "def rescale(x):\n", - " x = x.replace(np.inf, np.nan)\n", + " x = x.cast(pl.Float64).replace(float(\"inf\"), None)\n", " minimum = x.min()\n", " return (x - minimum) / (x.max() - minimum)\n", "\n", "\n", - "rescale(pd.Series([1, 2, 3, np.inf, 5]))" + "rescale(pl.Series([1, 2, 3, float(\"inf\"), 5], strict=False))" ] }, { @@ -356,7 +372,7 @@ "metadata": {}, "source": [ "This is an important part of the \"do not repeat yourself\" (or DRY) principle.\n", - "The more repetition you have in your code, the more places you need to remember to update when things change (and they always do!), and the more likely you are to create bugs over time." + "The more repetition you have in your code, the more places you need to remember to update when things change (and they always do!), and the more likely you are to create bugs over time.\n" ] }, { @@ -394,7 +410,7 @@ "text_input()\n", "```\n", "\n", - "A good example of this design is the **pandas** package: if you don't remember exactly which function you need to read in data, you can type `pd.read_` and jog your memory as the autocomplete brings up the options." + "A good example of this design is the **polars** package: if you don't remember exactly which function you need to read in data, you can type `pl.read_` and jog your memory as the autocomplete brings up the options.\n" ] }, { @@ -418,16 +434,15 @@ "print(str_variable)\n", "```\n", "\n", - "This would raise an error, because as far as your general code is concerned `str_variable` doesn't exist outside of the function. This is an example of a *local* variable, one that only exists within a function.\n", - "\n", + "This would raise an error, because as far as your general code is concerned `str_variable` doesn't exist outside of the function. This is an example of a _local_ variable, one that only exists within a function.\n", "\n", - "If you want to create variables inside a function and have them persist, you need to explicitly pass them out using, for example `return str_variable` like this:" + "If you want to create variables inside a function and have them persist, you need to explicitly pass them out using, for example `return str_variable` like this:\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "8725b986", + "id": "647da443", "metadata": {}, "outputs": [], "source": [ @@ -447,13 +462,13 @@ "source": [ "**Global**\n", "\n", - "A variable declared outside of a function is known as a global variable because it is accessible everywhere:" + "A variable declared outside of a function is known as a global variable because it is accessible everywhere:\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "b0a4fe15", + "id": "c5fb049d", "metadata": {}, "outputs": [], "source": [ @@ -470,9 +485,6 @@ } ], "metadata": { - "interpreter": { - "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" - }, "jupytext": { "cell_metadata_filter": "-all", "encoding": "# -*- coding: utf-8 -*-", @@ -480,7 +492,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "python4ds", "language": "python", "name": "python3" }, @@ -494,7 +506,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/index.md b/index.md index 965c550..6086806 100644 --- a/index.md +++ b/index.md @@ -5,29 +5,14 @@ aliases: # Welcome -[![DOI](https://zenodo.org/badge/496994611.svg)](https://zenodo.org/doi/10.5281/zenodo.10518241) ![GitHub Release](https://img.shields.io/github/v/release/aeturrell/python4DS) +This is the website for **Python for Data Science**, a book heavily inspired by the excellent [**R for Data Science (2e)**](https://r4ds.hadley.nz/). This book will teach you how to load up, transform, visualize, and begin to understand your data. The book aims to give you the skills you need to code for data science. It's suitable for people who have some familiarity with the ideas behind programming and coding but who don't yet know how to do data science. -This is the website for **Python for Data Science**, a book heavily inspired by the excellent [**R for Data Science (2e)**](https://r4ds.hadley.nz/). This book will teach you how to load up, transform, visualise, and begin to understand your data. The book aims to give you the skills you need to code for data science. It's suitable for people who have some familiarity with the ideas behind programming and coding but who don't yet know how to do data science. +This book teaches you how to do data science using one of the world's most popular programming languages, Python. While Python is a general-purpose language, meaning it is used for a wide range of tasks, it is also the most widely used language for data science (though note that both SQL and R are also used for data science). -This book teaches you how to do data science using one of the world's most popular programming languages, Python. While Python is a general purpose language, which means it is used for a wide range of tasks, it is also the most widely used language for data science (although note that both SQL and R are also used for data science). +__This [fork](https://github.com/aeturrell/python4DS) of a [fork](https://github.com/hadley/r4ds) focuses on Polars instead of Pandas for data wrangling.__ To begin your data science journey, head to the next page. ## Contributors to Python4DS Contributing is very much encouraged. If you're looking for content to implement or tweak, we aim to follow the structure and content of **R for Data Science (2e)** and you can find open [issues here](https://github.com/aeturrell/python4DS/issues). For larger contributions of content, it's probably best to check with other contributors first. - -We thank the following contributors: - -- [Arthur Turrell](https://aeturrell.com/), who has also contributed to [*Coding for Economists*](https://aeturrell.github.io/coding-for-economists) and wrote popular non-fiction book [*The Star Builders*](https://aeturrell.com/thestarbuilders/thestarbuilders.html) -- [Pietro Monticone](https://github.com/pitmonticone) -- [Antonio Mele](https://github.com/meleantonio) -- [Igor Alshannikov](https://github.com/alshan) -- [Umair Durrani](https://github.com/durraniu) -- [Zeki Akyol](https://github.com/zekiakyol) -- [Yiben Huang](https://github.com/yibenhuang) -- [William Chiu](https://github.com/crossxwill) -- [udurraniAtPresage](https://github.com/udurraniAtPresage) -- [Josh Holman](https://github.com/TheJolman) -- [Kenytt Avery](https://github.com/ProfAvery) -- [Bradley Phipps](https://github.com/hotshotberad) diff --git a/introduction.ipynb b/introduction.ipynb index 60bfa83..04c2ff9 100644 --- a/introduction.ipynb +++ b/introduction.ipynb @@ -187,7 +187,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/iteration.ipynb b/iteration.ipynb index fc1452a..b44f79a 100644 --- a/iteration.ipynb +++ b/iteration.ipynb @@ -21,7 +21,7 @@ "\n", "One tool for reducing duplication is functions, which reduce duplication by identifying repeated patterns of code and extract them out into independent pieces that can be easily reused and updated. Another tool for reducing duplication is *iteration*, which helps you when you need to do the same thing to multiple inputs: repeating the same operation on different columns, or on different datasets.\n", "\n", - "In this chapter you'll learn about iteration in three ways: explicit iteration, using for loops and while loops; iteration via comprehensions (eg list comprehensions); and iteration for **pandas** data frames." + "In this chapter you'll learn about iteration in three ways: explicit iteration, using for loops and while loops; iteration via comprehensions (eg list comprehensions); and iteration for **polars** data frames." ] }, { @@ -51,7 +51,7 @@ "source": [ "### Prerequisites\n", "\n", - "This chapter will use the **pandas** data analysis package." + "This chapter will use the **polars** data analysis package." ] }, { @@ -452,11 +452,13 @@ "id": "5ec0643e", "metadata": {}, "source": [ - "## Iteration with **pandas** Data Frames\n", + "## Iteration with **polars** Data Frames\n", "\n", - "For loops, while loops, and comprehensions all work on **pandas** data frames, but they are generally a bad way to get things done because they are slow and not memory efficient. To aid cases where iteration is needed, **pandas** has built-in methods for iteration depending on what you need to do.\n", + "For loops, while loops, and comprehensions can be used with data frames, but in **Polars**, they are even more strongly discouraged than in pandas. **Polars** is built on a columnar, vectorized, and expression-based engine, so row-by-row iteration breaks performance and prevents optimizations.\n", "\n", "These built-in methods for iteration have an overlap with what we've seen in @sec-data-transform but we'll dig a little deeper into `assign()`/assignment operations, `apply()`, and `eval()` here.\n", + "\n", + "Instead of iterating, **Polars** encourages you to use expressions and lazy evaluation, which are much faster and more memory efficient.\n", "\n" ] }, @@ -480,9 +482,9 @@ "outputs": [], "source": [ "import numpy as np\n", - "import pandas as pd\n", + "import polars as pl\n", "\n", - "df = pd.DataFrame(np.random.normal(size=(6, 4)), columns=[\"a\", \"b\", \"c\", \"d\"])\n", + "df = pl.DataFrame(np.random.normal(size=(6, 4)), schema=[\"a\", \"b\", \"c\", \"d\"])\n", "df" ] }, @@ -491,7 +493,7 @@ "id": "0e826ad5", "metadata": {}, "source": [ - "**pandas** has many built-in functions that are already built to iterate over rows and columns; for example, to compute the median of rows or columns respectively:" + "**polars** has built-in expressions designed to operate over columns and rows. For example, to compute the median:" ] }, { @@ -501,7 +503,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.median(axis=\"rows\") # can also use axis=1" + "df.select(pl.all().median())" ] }, { @@ -511,7 +513,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.median(axis=\"columns\") # can also use axis=0" + "df.select(pl.concat_list(pl.all()).list.median().alias(\"row_median\"))" ] }, { @@ -535,7 +537,7 @@ "def add_five_slow(df):\n", " for i in range(len(df)):\n", " for j in range(len(df.columns)):\n", - " df.iloc[i, j] = df.iloc[i, j] + 5\n", + " df[i, j] = df[i, j] + 5\n", "\n", "\n", "%timeit add_five_slow(df)" @@ -546,7 +548,7 @@ "id": "8246132e", "metadata": {}, "source": [ - "But to do this, every individual cell must be accessed and operated on—so it is very slow, taking milliseconds. **pandas** has far faster ways of performing the same operation. For simple operations on data frames with consistent type, you can simply add five to the whole data frame:" + "But to do this, every individual cell must be accessed and operated on—so it is very slow, taking milliseconds. **polars** has far faster ways of performing the same operation. For simple operations on data frames with consistent type, you can simply add five to the whole data frame:" ] }, { @@ -572,9 +574,9 @@ "id": "7313616e", "metadata": {}, "source": [ - "This also works on a per column basis, so you can do `df[\"a\"] = df[\"a\"] + 5` and so on.\n", + "This also works on a per column basis, so you can do `df.with_columns(pl.col(\"a\") + 5)` and so on.\n", "\n", - "These operations have equivalents using the `assign()` operator, which allows for *method chaining*; stringing multiple operations together. The `assign()` operator version of `df[\"new_a\"] = df[\"a\"] + 5` would be" + "These operations have equivalents using method chaining; stringing multiple operations together. The version of `df.with_columns(new_a = pl.col(\"a\") + 5)` would be:" ] }, { @@ -584,7 +586,7 @@ "metadata": {}, "outputs": [], "source": [ - "df = df.assign(new_a=lambda x: x[\"a\"] + 5)" + "df = df.with_columns(new_a=pl.col(\"a\") + 5)" ] }, { @@ -592,9 +594,11 @@ "id": "76aec162", "metadata": {}, "source": [ - "### Apply\n", + "### Expressions (Polars' Alternative to apply)\n", + "\n", + "What happens if you have a more complicated operation you want to perform? In pandas, you might reach for `apply()`. In **polars**, you almost never need an equivalent because its expression API is incredibly expressive.\n", "\n", - "What happens if you have a more complicated function you want to iterate over? This is where **pandas**' `apply()` comes in, and can be used with assignment. `apply()` can also be used across rows or columns. Like `assign()`, it can be combined with a lambda function and used with either the whole data frame or just a column (in which case no need to specify `axis=`)." + "Most \"complicated\" operations can be expressed directly using **polars'** built-in expressions:" ] }, { @@ -604,7 +608,16 @@ "metadata": {}, "outputs": [], "source": [ - "df.apply(lambda x: x[\"a\"] - x[\"new_a\"].mean() * x[\"c\"] / x[\"b\"], axis=1)" + "# Don't do this (slow, row-wise)\n", + "mean_new_a = df.select(pl.col(\"new_a\").mean()).item()\n", + "df.with_columns(\n", + " result=pl.struct([\"a\", \"b\", \"c\"]).map_elements(\n", + " lambda x: x[\"a\"] - mean_new_a * x[\"c\"] / x[\"b\"], return_dtype=pl.Float64\n", + " )\n", + ")\n", + "\n", + "# Do this instead (fast, vectorized)\n", + "df.with_columns(result=pl.col(\"a\") - pl.col(\"new_a\").mean() * pl.col(\"c\") / pl.col(\"b\"))" ] }, { @@ -612,24 +625,7 @@ "id": "78b558f4", "metadata": {}, "source": [ - "Note that this is just an example: you could still do this entire operation without using apply! But you will sometimes find yourself with cases where you do need to use it.\n", - "\n", - "Apply also works with functions, including user-defined functions:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "136d435d", - "metadata": {}, - "outputs": [], - "source": [ - "def complicated_function(x):\n", - " return x - x.mean()\n", - "\n", - "\n", - "df = df.apply(complicated_function, axis=1)\n", - "df" + "The first expression would work, but it evaluates the computation row by row using a **python lambda**, which is slow and prevents **polars** from optimizing the query. The second approach uses native expressions, allowing **polars** to execute the computation efficiently in a fully vectorized and optimized manner." ] }, { @@ -637,9 +633,7 @@ "id": "171be2c9", "metadata": {}, "source": [ - "### Eval(uate)\n", - "\n", - "`eval()` evaluates a string describing operations on DataFrame columns to create new columns. It operates on columns only, not rows or elements. Here's an example:" + "In **polars**, there's no `eval()` — you use expressions directly instead:\n" ] }, { @@ -649,7 +643,7 @@ "metadata": {}, "outputs": [], "source": [ - "df[\"ratio\"] = df.eval(\"a / new_a\")\n", + "df = df.with_columns((pl.col(\"a\") / pl.col(\"new_a\")).alias(\"ratio\"))\n", "df" ] }, @@ -658,7 +652,18 @@ "id": "8b275b5b", "metadata": {}, "source": [ - "Evaluate can also be used to create new boolean columns using, for example, a string `\"a > 0.5\"` in the above example." + "You can also create boolean columns the same way:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f301c8cb", + "metadata": {}, + "outputs": [], + "source": [ + "df = df.with_columns((pl.col(\"a\") > 0.5).alias(\"a_gt_0.5\"))\n", + "df" ] } ], @@ -687,7 +692,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/joins.ipynb b/joins.ipynb index e58d2d3..d494d84 100644 --- a/joins.ipynb +++ b/joins.ipynb @@ -255,7 +255,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/missing-values.ipynb b/missing-values.ipynb index 262f015..67e95ef 100644 --- a/missing-values.ipynb +++ b/missing-values.ipynb @@ -596,7 +596,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/numbers.ipynb b/numbers.ipynb index 388db20..0b96fbb 100644 --- a/numbers.ipynb +++ b/numbers.ipynb @@ -792,7 +792,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/prerequisites.ipynb b/prerequisites.ipynb index 68bea6d..07119a8 100644 --- a/prerequisites.ipynb +++ b/prerequisites.ipynb @@ -305,7 +305,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.0" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/pyproject.toml b/pyproject.toml index 016e885..7740186 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,12 @@ [project] name = "python4ds" -version = "1.0.5" +version = "0.0.1" description = "The online book that teaches you how to use Python for data science." readme = "README.md" requires-python = ">=3.12.0,<3.13" dependencies = [ "beautifulsoup4>=4.12.3", + "fastexcel>=0.19.0", "graphviz>=0.20.3", "ibis-framework[sqlite]>=9.5.0", "ipykernel>=6.29.5", @@ -36,6 +37,7 @@ dependencies = [ "toml>=0.10.2", "watermark>=2.5.0", "wbgapi>=1.0.14", + "xlsxwriter>=3.2.0", "yfinance>=1.2.1", ] diff --git a/rectangling.ipynb b/rectangling.ipynb index f0bcf47..2ddf375 100644 --- a/rectangling.ipynb +++ b/rectangling.ipynb @@ -41,7 +41,7 @@ "source": [ "### Prerequisites\n", "\n", - "This chapter will use the **pandas** data analysis package." + "This chapter will use the **polars** data analysis package.\n" ] }, { @@ -51,7 +51,7 @@ "source": [ "## Lists\n", "\n", - "Lists are a really useful way to work with lots of data at once. They're defined with square brackets, with entries separated by commas. " + "Lists are a really useful way to work with lots of data at once. They're defined with square brackets, with entries separated by commas.\n" ] }, { @@ -70,7 +70,7 @@ "id": "29b10d07", "metadata": {}, "source": [ - "You can also construct them by appending entries:" + "You can also construct them by appending entries:\n" ] }, { @@ -89,7 +89,7 @@ "id": "d8d4f6ed", "metadata": {}, "source": [ - "And you can access earlier entries using an index, which begins at 0 and ends at one less than the length of the list (this is the convention in many programming languages). For instance, to print specific entries at the start, using `0`, and end, using `-1`:" + "And you can access earlier entries using an index, which begins at 0 and ends at one less than the length of the list (this is the convention in many programming languages). For instance, to print specific entries at the start, using `0`, and end, using `-1`:\n" ] }, { @@ -110,7 +110,7 @@ "source": [ "::: {.callout-tip title=\"Exercise\"}\n", "How might you access the penultimate entry in a list object if you didn't know how many elements it had?\n", - ":::" + ":::\n" ] }, { @@ -118,7 +118,7 @@ "id": "6aea9157", "metadata": {}, "source": [ - "As well as accessing positions in lists using indexing, you can use *slices* on lists. This uses the colon character, `:`, to stand in for 'from the beginning' or 'until the end' (when only appearing once). For instance, to print just the last two entries, we would use the index `-2:` to mean from the second-to-last onwards. Here are two distinct examples: getting the first three and last three entries to be successively printed:" + "As well as accessing positions in lists using indexing, you can use _slices_ on lists. This uses the colon character, `:`, to stand in for 'from the beginning' or 'until the end' (when only appearing once). For instance, to print just the last two entries, we would use the index `-2:` to mean from the second-to-last onwards. Here are two distinct examples: getting the first three and last three entries to be successively printed:\n" ] }, { @@ -137,7 +137,7 @@ "id": "c82b5c4a", "metadata": {}, "source": [ - "Slicing can be even more elaborate than that because we can jump entries using a second colon. Here's a full example that begins at the second entry (remember the index starts at 0), runs up until the second-to-last entry (exclusive), and jumps every other entry inbetween (range just produces a list of integers from the value to one less than the last):" + "Slicing can be even more elaborate than that because we can jump entries using a second colon. Here's a full example that begins at the second entry (remember the index starts at 0), runs up until the second-to-last entry (exclusive), and jumps every other entry inbetween (range just produces a list of integers from the value to one less than the last):\n" ] }, { @@ -159,7 +159,7 @@ "id": "813e09bc", "metadata": {}, "source": [ - "A handy trick is that you can print a reversed list entirely using double colons:" + "A handy trick is that you can print a reversed list entirely using double colons:\n" ] }, { @@ -179,7 +179,7 @@ "source": [ "::: {.callout-tip title=\"Exercise\"}\n", "Slice the `list_example` from earlier to get only the first five entries.\n", - ":::" + ":::\n" ] }, { @@ -187,7 +187,7 @@ "id": "b6ff3ca4", "metadata": {}, "source": [ - "What's amazing about lists is that they can hold any type, including other lists! Here's a valid example of a list that's got a lot going on:" + "What's amazing about lists is that they can hold any type, including other lists! Here's a valid example of a list that's got a lot going on:\n" ] }, { @@ -217,7 +217,7 @@ "source": [ "### Hierarchical Data in Lists\n", "\n", - "Because lists can contain more lists (and so on), they can be used to put hierachical data in. Let's take a look at an example:" + "Because lists can contain more lists (and so on), they can be used to put hierachical data in. Let's take a look at an example:\n" ] }, { @@ -236,7 +236,7 @@ "id": "57a81b53", "metadata": {}, "source": [ - "Now, say we wanted to reduce this to a single list. We can do it with a *list comprehension*:" + "Now, say we wanted to reduce this to a single list. We can do it with a _list comprehension_:\n" ] }, { @@ -254,7 +254,7 @@ "id": "8e96185a", "metadata": {}, "source": [ - "What we're saying here is take all of the values of every little list and put them into a single list." + "What we're saying here is take all of the values of every little list and put them into a single list.\n" ] }, { @@ -264,7 +264,7 @@ "source": [ "### From Lists to Data Frames\n", "\n", - "Occassionally, you'll have data in lists that you wish to turn into a data frame. For example, perhaps you have a list of lists like this:" + "Occassionally, you'll have data in lists that you wish to turn into a data frame. For example, perhaps you have a list of lists like this:\n" ] }, { @@ -282,7 +282,7 @@ "id": "fcfc2d3c", "metadata": {}, "source": [ - "You can pass this straight into a constructor for a data frame as the `data=` keyword argument (adding in other info as necessary). Note that this is four lists of three entries, so the inner loop has entries in 0 to 2... it is this inner loop that will be used as the *rows* of any data frame with the number of entries in each inner list equal to the number of *columns*." + "You can pass this straight into a constructor for a data frame as the `data=` keyword argument (adding in other info as necessary). Note that this is four lists of three entries, so the inner loop has entries in 0 to 2... it is this inner loop that will be used as the _rows_ of any data frame with the number of entries in each inner list equal to the number of _columns_.\n" ] }, { @@ -292,9 +292,9 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import polars as pl\n", "\n", - "pd.DataFrame(data=list_of_lists, columns=[\"a\", \"b\", \"c\"])" + "df = pl.DataFrame(data=list_of_lists, schema=[\"a\", \"b\", \"c\", \"d\"])" ] }, { @@ -302,7 +302,7 @@ "id": "cc797c89", "metadata": {}, "source": [ - "There's one more trick to show you: explode. This is useful when you have data that has more than one level of list depth. Let's say you read in some data with a complex hierarchical structure like this:" + "There's one more trick to show you: explode. This is useful when you have data that has more than one level of list depth. Let's say you read in some data with a complex hierarchical structure like this:\n" ] }, { @@ -312,12 +312,13 @@ "metadata": {}, "outputs": [], "source": [ - "df = pd.DataFrame(\n", + "df = pl.DataFrame(\n", " {\n", - " \"alpha\": [[0, 1, 2], \"foo\", [], [3, 4]],\n", - " \"beta\": 1,\n", - " \"gamma\": [[\"a\", \"b\", \"c\"], pd.NA, [], [\"d\", \"e\"]],\n", - " }\n", + " \"alpha\": [[\"0,1,2\"], \"foo\", [], [\"3,4\"]],\n", + " \"beta\": [1, 1, 1, 1],\n", + " \"gamma\": [[\"a\", \"b\", \"c\"], None, [], [\"d\", \"e\"]],\n", + " },\n", + " strict=False,\n", ")\n", "df" ] @@ -327,7 +328,7 @@ "id": "91bb97aa", "metadata": {}, "source": [ - "We have multiple rows and columns that contain lists. In some situations, it's fine to have a list in a column but here it's probably not as it's mixed in with other types of data. We can use `explode()` to split out the columns further length-wise" + "We have multiple rows and columns that contain lists. In some situations, it's fine to have a list in a column but here it's probably not as it's mixed in with other types of data. We can use `explode()` to split out the columns further length-wise\n" ] }, { @@ -337,7 +338,7 @@ "metadata": {}, "outputs": [], "source": [ - "df.explode(\"alpha\")" + "df.explode(\"gamma\")" ] }, { @@ -352,7 +353,7 @@ "The table below compares the different data types found in Python and JSON.\n", "\n", "| JSON OBJECT | PYTHON OBJECT |\n", - "|---------------|---------------|\n", + "| ------------- | ------------- |\n", "| object | dict |\n", "| array | list |\n", "| string | str |\n", @@ -362,9 +363,9 @@ "| true | True |\n", "| false | False |\n", "\n", - "There are typically two operations you may want to do with JSON data: 1) turn JSON data in a Python object (eg JSON to Python dictionary) or vice versa (known as deserialisation and serialisation respectively); and 2) converting a deserialised object into a *different* kind of Python object.\n", + "There are typically two operations you may want to do with JSON data: 1) turn JSON data in a Python object (eg JSON to Python dictionary) or vice versa (known as deserialisation and serialisation respectively); and 2) converting a deserialised object into a _different_ kind of Python object.\n", "\n", - "Let's look at each in turn." + "Let's look at each in turn.\n" ] }, { @@ -378,7 +379,7 @@ "\n", "#### From the Web\n", "\n", - "We'll get some JSON data from an API. Let's grab the latest UK unemployment data (timeseries code \"MGSX\" and dataset code \"LMS\")." + "We'll get some JSON data from an API. Let's grab the latest UK unemployment data (timeseries code \"MGSX\" and dataset code \"LMS\").\n" ] }, { @@ -401,7 +402,7 @@ "id": "051d3b4a", "metadata": {}, "source": [ - "Let's check what type we got:" + "Let's check what type we got:\n" ] }, { @@ -421,7 +422,7 @@ "source": [ "As expected, the JSON data has automatically been read in as a dictionary—but be wary that the fields have been read in as text rather than numbers, datetimes, and other specific data types.\n", "\n", - "We could print the whole object out but that would take up a lot of space; instead let's look at a couple of entries under the \"months\" key." + "We could print the whole object out but that would take up a lot of space; instead let's look at a couple of entries under the \"months\" key.\n" ] }, { @@ -441,7 +442,7 @@ "source": [ "#### From a File or Stream\n", "\n", - "For this exercise, you'll need to download the JSON file 'cakes.json' from the [data folder of the repository](https://github.com/aeturrell/python4DS/tree/main/data) associated with this book and save it in a sub-folder called \"data\". We can take a peek at the data using the terminal (which is what the preceeding exclamation mark means):" + "For this exercise, you'll need to download the JSON file 'cakes.json' from the [data folder of the repository](https://github.com/aeturrell/python4DS/tree/main/data) associated with this book and save it in a sub-folder called \"data\". We can take a peek at the data using the terminal (which is what the preceeding exclamation mark means):\n" ] }, { @@ -467,7 +468,7 @@ "id": "0c664ab6", "metadata": {}, "source": [ - "We use the built-in **json** library to read this into Python (you could also use a file path here—more on how in a moment):" + "We use the built-in **json** library to read this into Python (you could also use a file path here—more on how in a moment):\n" ] }, { @@ -488,7 +489,7 @@ "id": "df41f92b", "metadata": {}, "source": [ - "Note that not everything is the same in going from JSON text to a Python dictionary: JSON uses `null` rather than `None`, won't accept trailing commas at the end of lists, and has basic types that are lists, strings (and all keys must be strings), numbers, booleans, and nulls. Let's now see how to write a Python dictionary back to a JSON, perhaps for writing to file:" + "Note that not everything is the same in going from JSON text to a Python dictionary: JSON uses `null` rather than `None`, won't accept trailing commas at the end of lists, and has basic types that are lists, strings (and all keys must be strings), numbers, booleans, and nulls. Let's now see how to write a Python dictionary back to a JSON, perhaps for writing to file:\n" ] }, { @@ -507,7 +508,7 @@ "id": "5f9445b8", "metadata": {}, "source": [ - "To write to a file, you would use the pattern:" + "To write to a file, you would use the pattern:\n" ] }, { @@ -518,7 +519,7 @@ "```python\n", "with open('data/json_data_output.json', 'w') as outfile:\n", " json.dump(json_stream, outfile)\n", - "```" + "```\n" ] }, { @@ -530,7 +531,7 @@ "\n", "```python\n", "json.load(open(\"data/json_data_output.json\"))\n", - "```" + "```\n" ] }, { @@ -540,7 +541,7 @@ "source": [ "### From JSON data to Data Frame\n", "\n", - "**pandas** has lots of options for turning JSON or dictionary data into a data frame. You do need to think a little bit about the structure of the data underneath though:\n" + "**polars** has lots of options for turning JSON or dictionary data into a data frame. You do need to think a little bit about the structure of the data underneath though:\n" ] }, { @@ -550,9 +551,9 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import polars as pl\n", "\n", - "pd.DataFrame(result[\"toppings\"], columns=[\"id\", \"type\"])" + "df = pl.DataFrame(result[\"toppings\"], schema=[\"id\", \"type\"])" ] }, { @@ -560,7 +561,7 @@ "id": "a1346020", "metadata": {}, "source": [ - "The web-scraped data we downloaded earlier had a more complicated structure, but **pandas** has a `json_normalize()` function that can cope with this. For example, with the following data, there are many missing entries but `json_normalize()` can still parse it into a Data Frame." + "The web-scraped data we downloaded earlier had a more complicated structure, but **polars** has a `json_normalize()` function that can cope with this. For example, with the following data, there are many missing entries but `json_normalize()` can still parse it into a Data Frame.\n" ] }, { @@ -575,7 +576,7 @@ " {\"name\": {\"given\": \"Mark\", \"family\": \"Regner\"}},\n", " {\"id\": 2, \"name\": \"Faye Raker\"},\n", "]\n", - "pd.json_normalize(data)" + "pl.json_normalize(data)" ] }, { @@ -583,7 +584,7 @@ "id": "7eaf00e1", "metadata": {}, "source": [ - "And we can control the level that properties like 'name' are split out to as well (you can check out more options over at the [**pandas** documentation](https://pandas.pydata.org/docs/reference/api/pandas.json_normalize.html))" + "And we can control the level that properties like 'name' are split out to as well (you can check out more options over at the [**polars** documentation](https://docs.pola.rs/api/python/stable/reference/api/polars.json_normalize.html))\n" ] }, { @@ -593,7 +594,7 @@ "metadata": {}, "outputs": [], "source": [ - "pd.json_normalize(data, max_level=0)" + "pl.json_normalize(data, max_level=0)" ] }, { @@ -601,7 +602,7 @@ "id": "78d637e5", "metadata": {}, "source": [ - "As well as the JSON normalise function, **pandas** has a `from_dict()` method to work with simpler dictionary objects." + "As well as the JSON normalise function, **polars** has a `from_dict()` method to work with simpler dictionary objects.\n" ] } ], @@ -613,7 +614,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "python4ds", "language": "python", "name": "python3" }, @@ -627,7 +628,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/regex.ipynb b/regex.ipynb index a8055fc..520a8c8 100644 --- a/regex.ipynb +++ b/regex.ipynb @@ -233,7 +233,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" } }, "nbformat": 4, diff --git a/spreadsheets.ipynb b/spreadsheets.ipynb index 12f1672..6d62900 100644 --- a/spreadsheets.ipynb +++ b/spreadsheets.ipynb @@ -11,7 +11,7 @@ "\n", "This chapter will show you how to work with spreadsheets, for example Microsoft Excel files, in Python. We already saw how to import csv (and tsv) files in @sec-data-import. In this chapter we will introduce you to tools for working with data in Excel spreadsheets and Google Sheets.\n", "\n", - "If you or your collaborators are using spreadsheets for organising data that will be ingested by an analytical tool like Python, we recommend reading the paper \"Data Organization in Spreadsheets\" by Karl Broman and Kara Woo {cite}`broman2018data`. The best practices presented in this paper will save you much headache down the line when you import the data from a spreadsheet into Python to analyse and visualise. (For spreadsheets that are meant to be read by humans, we recommend the [good practice tables](https://github.com/best-practice-and-impact/gptables) package.)" + "If you or your collaborators are using spreadsheets for organising data that will be ingested by an analytical tool like Python, we recommend reading the paper \"Data Organization in Spreadsheets\" by Karl Broman and Kara Woo {cite}`broman2018data`. The best practices presented in this paper will save you much headache down the line when you import the data from a spreadsheet into Python to analyse and visualise. (For spreadsheets that are meant to be read by humans, we recommend the [good practice tables](https://github.com/best-practice-and-impact/gptables) package.)\n" ] }, { @@ -41,7 +41,7 @@ "source": [ "### Prerequisites\n", "\n", - "You will need to install the **pandas** package for this chapter. You will also need to install the **openpyxl** package by running `uv add openpyxl` in the terminal." + "You will need the **polars** package for this chapter. Install **fastexcel** so `read_excel()` can use the default (fast) engine, **xlsxwriter** for `write_excel()`, and **openpyxl** if you want to use the **openpyxl** engine explicitly (`uv add fastexcel xlsxwriter openpyxl`).\n" ] }, { @@ -51,11 +51,11 @@ "source": [ "## Reading Excel (and Similar) Files\n", "\n", - "**pandas** can read in xls, xlsx, xlsm, xlsb, odf, ods, and odt files from your local filesystem or from a URL. It also supports an option to read a single sheet or a list of sheets.\n", + "**polars** can read in xls, xlsx, xlsm, xlsb, odf, ods, and odt files from your local filesystem or from a URL. It also supports an option to read a single sheet or a list of sheets. The default reader uses **fastexcel** (Rust-backed, via the **fastexcel** Python package); you can also select other engines such as **openpyxl** when you need engine-specific options.\n", "\n", "To show how this works, we'll work with an example spreadsheet called \"students.xlsx\". The figure below shows what the spreadsheet looks like.\n", "\n", - "![A look at the students spreadsheet in Excel. The spreadsheet contains information on 6 students, their ID, full name, favourite food, meal plan, and age.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-students.png)" + "![A look at the students spreadsheet in Excel. The spreadsheet contains information on 6 students, their ID, full name, favourite food, meal plan, and age.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-students.png)\n" ] }, { @@ -63,7 +63,7 @@ "id": "29f2f4e0", "metadata": {}, "source": [ - "The first argument to `pd.read_excel()` is the path to the file to read. If you have downloaded the [file]() onto your computer and put it in a subfolder called \"data\" then you would want to use the path \"data/students.xlsx\" but we can also load it directly from the URL." + "The first argument to `pl.read_excel()` is the path to the file to read. If you have downloaded the [file]() onto your computer and put it in a subfolder called \"data\" then you would want to use the path \"data/students.xlsx\" but we can also load it directly from the URL.\n" ] }, { @@ -73,10 +73,10 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import polars as pl\n", "\n", - "students = pd.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/students.xlsx\"\n", + "students = pl.read_excel(\n", + " \"data/students.xlsx\",\n", ")\n", "students" ] @@ -88,7 +88,7 @@ "source": [ "We have six students in the data and five variables on each student. However there are a few things we might want to address in this dataset:\n", "\n", - "- The column names are all over the place. You can provide column names that follow a consistent format; we recommend `snake_case` using the `names` argument.\n" + "- The column names are all over the place. You can rename them to follow a consistent format; we recommend `snake_case`. If you want to replace **every** column name in order, assigning to `students.columns` is clear and short. If you only rename some columns, use `.rename({\"Old Name\": \"new_name\", ...})` with the exact strings from the sheet.\n" ] }, { @@ -98,10 +98,14 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/students.xlsx\",\n", - " names=[\"student_id\", \"full_name\", \"favourite_food\", \"meal_plan\", \"age\"],\n", - ")" + "students.columns = [\n", + " \"student_id\",\n", + " \"full_name\",\n", + " \"favourite_food\",\n", + " \"meal_plan\",\n", + " \"age\",\n", + "]\n", + "students" ] }, { @@ -109,8 +113,7 @@ "id": "bb07ad4f", "metadata": {}, "source": [ - "\n", - "- `age` is read in as a column of objects, but it really should be numeric. Just like with `read_csv()`, you can supply a `dtype` argument to `read_excel()` and specify the data types for the columns of data you read in. Your options include `\"boolean\"`, `\"int\"`, `\"float\"`, `\"datetime\"`, `\"string\"`, and more. But we can see right away that this isn't going to work with the \"age\" column as it mixes numbers and text: so we first need to map its text to numbers." + "- `age` may be inferred as strings (for example **Utf8**) when the column mixes numeric values and text, but we want it numeric. Just like with `read_csv()`, you can supply a `schema_overrides` argument to `read_excel()` and specify Polars data types for the columns you read in (for example `pl.Int64`, `pl.Utf8`, `pl.Boolean`, `pl.Datetime`, and more). That still will not fix a value like `\"five\"` until we map it to a number first.\n" ] }, { @@ -120,11 +123,15 @@ "metadata": {}, "outputs": [], "source": [ - "students = pd.read_excel(\n", - " \"data/students.xlsx\",\n", - " names=[\"student_id\", \"full_name\", \"favourite_food\", \"meal_plan\", \"age\"],\n", - ")\n", - "students[\"age\"] = students[\"age\"].replace(\"five\", 5)\n", + "students = pl.read_excel(\"data/students.xlsx\")\n", + "students.columns = [\n", + " \"student_id\",\n", + " \"full_name\",\n", + " \"favourite_food\",\n", + " \"meal_plan\",\n", + " \"age\",\n", + "]\n", + "students = students.with_columns(pl.col(\"age\").replace({\"five\": 5}))\n", "students" ] }, @@ -133,7 +140,7 @@ "id": "c8a07159", "metadata": {}, "source": [ - "Okay, now we can apply the data types." + "Okay, now we can apply the data types.\n" ] }, { @@ -143,16 +150,16 @@ "metadata": {}, "outputs": [], "source": [ - "students = students.astype(\n", - " {\n", - " \"student_id\": \"Int64\",\n", - " \"full_name\": \"string\",\n", - " \"favourite_food\": \"string\",\n", - " \"meal_plan\": \"category\",\n", - " \"age\": \"Int64\",\n", - " }\n", + "students = students.with_columns(\n", + " [\n", + " pl.col(\"student_id\").cast(pl.Int64),\n", + " pl.col(\"full_name\").cast(pl.Utf8),\n", + " pl.col(\"favourite_food\").cast(pl.Utf8),\n", + " pl.col(\"meal_plan\").cast(pl.Categorical),\n", + " pl.col(\"age\").cast(pl.Int64),\n", + " ]\n", ")\n", - "students.info()" + "students.schema" ] }, { @@ -160,7 +167,7 @@ "id": "362ff5a5", "metadata": {}, "source": [ - "It took multiple steps and trial-and-error to load the data in exactly the format we want, and this is not unexpected. Data science is an iterative process. There is no way to know exactly what the data will look like until you load it and take a look at it. The general pattern we used is load the data, take a peek, make adjustments to your code, load it again, and repeat until you're happy with the result." + "It took multiple steps and trial-and-error to load the data in exactly the format we want, and this is not unexpected. Data science is an iterative process. There is no way to know exactly what the data will look like until you load it and take a look at it. The general pattern we used is load the data, take a peek, make adjustments to your code, load it again, and repeat until you're happy with the result.\n" ] }, { @@ -174,7 +181,7 @@ "\n", "![A look at the penguins spreadsheet in Excel. The spreadsheet contains has three sheets: Torgersen Island, Biscoe Island, and Dream Island.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-penguins-islands.png)\n", "\n", - "You can read a single sheet using the following command (so as not to show the whole file, we'll use `.head()` to just show the first 5 rows):" + "You can read a single sheet using the following command (so as not to show the whole file, we'll use `.head()` to just show the first 5 rows):\n" ] }, { @@ -184,8 +191,8 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/penguins.xlsx\",\n", + "pl.read_excel(\n", + " \"data/penguins.xlsx\",\n", " sheet_name=\"Torgersen Island\",\n", ").head()" ] @@ -195,9 +202,9 @@ "id": "641f6831", "metadata": {}, "source": [ - "Now this relies on us knowing the names of the sheets in advance. There will be situations where you can to read in data without peeking into the Excel spreadsheet. To read all sheets in, use `sheet_name=None`. The object that's created is a dictionary with key value pairs that are sheet names and data frames respectively. Let's look at the second key value pair (note that we have to convert the keys() and values() objects to list to then retrieve the second element of each using a subscript, ie `list(dictionary.keys())[]`).\n", + "Now this relies on us knowing the names of the sheets in advance. There will be situations where you want to read in data without peeking into the Excel spreadsheet. To read all sheets in Polars, use `sheet_id=0` (or `sheet_name=None`, which also works in recent versions of Polars). The object that’s created is a dictionary where the keys are the sheet names and the values are Polars DataFrames. To access a specific sheet, you can convert the keys() or values() to a list and then index into it, ie `list(dictionary.keys())[]` .\n", "\n", - "To give a sense of how this works, let's first print all of the retrieved keys:" + "To give a sense of how this works, let's first print all of the retrieved keys:\n" ] }, { @@ -207,9 +214,9 @@ "metadata": {}, "outputs": [], "source": [ - "penguins_dict = pd.read_excel(\n", - " \"https://github.com/aeturrell/python4DS/raw/main/data/penguins.xlsx\",\n", - " sheet_name=None,\n", + "penguins_dict = pl.read_excel(\n", + " \"data/penguins.xlsx\",\n", + " sheet_id=0,\n", ")\n", "print([x for x in penguins_dict.keys()])" ] @@ -219,7 +226,7 @@ "id": "076f1ebe", "metadata": {}, "source": [ - "Now let's show the second entry data frame" + "Now let's show the second entry data frame\n" ] }, { @@ -238,7 +245,7 @@ "id": "536ab4bb", "metadata": {}, "source": [ - "What we really want is these three *consistent* datasets to be in the *same* single data frame. For this, we can use the `pd.concat()` function. This concatenates any given iterable of data frames." + "What we really want is these three _consistent_ datasets to be in the _same_ single data frame. For this, we can use the `pl.concat()` function. This concatenates any given iterable of data frames.\n" ] }, { @@ -248,7 +255,7 @@ "metadata": {}, "outputs": [], "source": [ - "penguins = pd.concat(penguins_dict.values(), axis=0)\n", + "penguins = pl.concat(penguins_dict.values())\n", "penguins" ] }, @@ -263,8 +270,7 @@ "\n", "The figure below shows such a spreadsheet: in the middle of the sheet is what looks like a data frame but there is extraneous text in cells above and below the data.\n", "\n", - "![A look at the deaths spreadsheet in Excel. The spreadsheet has four rows on top that contain non-data information; the text 'For the same of consistency in the data layout, which is really a beautiful thing, I will keep making notes up here.' is spread across cells in these top four rows. Then, there is a data frame that includes information on deaths of 10 famous people, including their names, professions, ages, whether they have kids or not, date of birth and death. At the bottom, there are four more rows of non-data information; the text 'This has been really fun, but we're signing off now!' is spread across cells in these bottom four rows.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-deaths.png)\n", - "\n" + "![A look at the deaths spreadsheet in Excel. The spreadsheet has four rows on top that contain non-data information; the text 'For the same of consistency in the data layout, which is really a beautiful thing, I will keep making notes up here.' is spread across cells in these top four rows. Then, there is a data frame that includes information on deaths of 10 famous people, including their names, professions, ages, whether they have kids or not, date of birth and death. At the bottom, there are four more rows of non-data information; the text 'This has been really fun, but we're signing off now!' is spread across cells in these bottom four rows.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-deaths.png)\n" ] }, { @@ -274,8 +280,7 @@ "source": [ "This spreadsheet can be downloaded from [here](https://github.com/aeturrell/python4DS/tree/main/data) or you can load it directly from a URL. If you want to load it from your own computer's disk, you'll need to save it in a sub-folder called \"data\" first.\n", "\n", - "\n", - "The top three rows and the bottom four rows are not part of the data frame. We could skip the top three rows with `skiprows`. Note that we set `skiprows=4` since the fourth row contains column names, not the data.\n" + "The top three rows and the bottom four rows are not part of the data frame. We could skip the top three rows by passing `read_options` to `read_excel()`. Note that we set `skip_rows=4` since the fourth row contains column names, not the data.\n" ] }, { @@ -285,7 +290,10 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_excel(\"data/deaths.xlsx\", skiprows=4)" + "pl.read_excel(\n", + " \"data/deaths.xlsx\",\n", + " read_options={\"skip_rows\": 4},\n", + ")" ] }, { @@ -293,7 +301,7 @@ "id": "a1a8c3ca", "metadata": {}, "source": [ - "We could also set `nrows` to omit the extraneous rows at the bottom (another option would to be to skip a set number of rows at the end using `skipfooter`)." + "We could also set `n_rows` inside `read_options` to omit the extraneous rows at the bottom (another option would be to skip a set number of rows at the end using `skip_footer` in `read_options`, depending on the engine).\n" ] }, { @@ -303,7 +311,10 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_excel(\"data/deaths.xlsx\", skiprows=4, nrows=10)" + "pl.read_excel(\n", + " \"data/deaths.xlsx\",\n", + " read_options={\"skip_rows\": 4, \"n_rows\": 10},\n", + ")" ] }, { @@ -317,20 +328,20 @@ "\n", "The underlying data in Excel spreadsheets is more complex. A cell can be one of five things:\n", "\n", - "- A logical, like TRUE / FALSE\n", + "- A logical, like TRUE / FALSE\n", "\n", - "- A number, like \"10\" or \"10.5\"\n", + "- A number, like \"10\" or \"10.5\"\n", "\n", - "- A date, which can also include time like \"11/1/21\" or \"11/1/21 3:00 PM\"\n", + "- A date, which can also include time like \"11/1/21\" or \"11/1/21 3:00 PM\"\n", "\n", - "- A string, like \"ten\"\n", + "- A string, like \"ten\"\n", "\n", - "- A currency, which allows numeric values in a limited range and four decimal digits of fixed precision\n", + "- A currency, which allows numeric values in a limited range and four decimal digits of fixed precision\n", "\n", "When working with spreadsheet data, it's important to keep in mind that how the underlying data is stored can be very different than what you see in the cell. For example, Excel has no notion of an integer. All numbers are stored as floating points (real number), but you can choose to display the data with a customizable number of decimal points. Similarly, dates are actually stored as numbers, specifically the number of seconds since January 1, 1970. You can customize how you display the date by applying formatting in Excel. Confusingly, it's also possible to have something that looks like a number but is actually a string (e.g. type `'10` into a cell in Excel).\n", "\n", - "These differences between how the underlying data are stored vs. how they're displayed can cause surprises when the data are loaded into analytical tools such as **pandas**. By default, **pandas** will guess the data type in a given column.\n", - "A recommended workflow is to let **pandas** guess the column types initially, inspect them, and then change any data types that you want to." + "These differences between how the underlying data are stored vs. how they're displayed can cause surprises when the data are loaded into analytical tools such as **polars**. By default, **polars** will guess the data type in a given column.\n", + "A recommended workflow is to let **polars** guess the column types initially, inspect them, and then change any data types that you want to.\n" ] }, { @@ -340,7 +351,7 @@ "source": [ "## Writing to Excel\n", "\n", - "Let's create a small data frame that we can then write out. Note that `item` is a category and `quantity` is an integer." + "Let's create a small data frame that we can then write out. Note that `item` is a category and `quantity` is an integer.\n" ] }, { @@ -350,8 +361,11 @@ "metadata": {}, "outputs": [], "source": [ - "bake_sale = pd.DataFrame(\n", - " {\"item\": pd.Categorical([\"brownie\", \"cupcake\", \"cookie\"]), \"quantity\": [10, 5, 8]}\n", + "bake_sale = pl.DataFrame(\n", + " {\n", + " \"item\": pl.Series([\"brownie\", \"cupcake\", \"cookie\"], dtype=pl.Categorical),\n", + " \"quantity\": [10, 5, 8],\n", + " }\n", ")\n", "bake_sale" ] @@ -361,17 +375,17 @@ "id": "345bca3d", "metadata": {}, "source": [ - "You can write data back to disk as an Excel file using the `.to_excel()` function. The `index=False` keyword argument just writes the two columns without the index that was automatically added in the last step." + "You can write data back to disk as an Excel file using the `.write_excel()` method. Polars does not use a row index like pandas, so only the columns in the DataFrame are written by default.\n" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "id": "1fc17141", "metadata": {}, + "outputs": [], "source": [ - "```python\n", - "bake_sale.to_excel(\"data/bake_sale.xlsx\", index=False)\n", - "```" + "bake_sale.write_excel(\"data/bake_sale.xlsx\")" ] }, { @@ -381,7 +395,7 @@ "source": [ "The figure below shows what the data looks like in Excel.\n", "\n", - "![Bake sale data frame created earlier in Excel.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-bake-sale.png)" + "![Bake sale data frame created earlier in Excel.](https://github.com/hadley/r4ds/raw/main/screenshots/import-spreadsheets-bake-sale.png)\n" ] }, { @@ -389,7 +403,7 @@ "id": "8d555c84", "metadata": {}, "source": [ - "Just like reading from a CSV, information on data type is lost when we read the data back in—you can see this is you read the data back in and check the `info` for the data types. Although we kept `int64` because **pandas** recognise that the second column was of integer type, we lost the categorical data type for \"item\". This data type loss makes Excel files unreliable for caching interim results." + "Just like reading from a CSV, information on data type is lost when we read the data back in—you can see this if you read the data back in and check the `schema` for the data types. Although we kept `Int64` because **polars** recognised that the second column was of integer type, we lost the categorical data type for \"item\". This data type loss makes Excel files unreliable for caching interim results.\n" ] }, { @@ -399,7 +413,7 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_excel(\"data/bake_sale.xlsx\").info()" + "pl.read_excel(\"data/bake_sale.xlsx\").schema" ] }, { @@ -409,14 +423,11 @@ "source": [ "### Formatted Output\n", "\n", - "If you need more formatting options and more control over how you write spreadsheets, check out the documentation for [openpyxl](https://openpyxl.readthedocs.io/) which can do pretty much everything you imagine. Generally, releasing data in spreadsheets is not the best option: but if you do want to release data in spreadsheets according to best practice, then check out [gptables](https://gptables.readthedocs.io/)." + "If you need more formatting options and more control over how you write spreadsheets, check out the documentation for [openpyxl](https://openpyxl.readthedocs.io/) which can do pretty much everything you imagine. Generally, releasing data in spreadsheets is not the best option: but if you do want to release data in spreadsheets according to best practice, then check out [gptables](https://gptables.readthedocs.io/).\n" ] } ], "metadata": { - "interpreter": { - "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" - }, "jupytext": { "cell_metadata_filter": "-all", "encoding": "# -*- coding: utf-8 -*-", @@ -424,7 +435,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "python4ds", "language": "python", "name": "python3" }, @@ -438,7 +449,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/strings.ipynb b/strings.ipynb index e401876..b1a00fe 100644 --- a/strings.ipynb +++ b/strings.ipynb @@ -1089,7 +1089,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/uv.lock b/uv.lock index 8316606..8e7d9ad 100644 --- a/uv.lock +++ b/uv.lock @@ -354,6 +354,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/fd/afcd0496feca3276f509df3dbd5dae726fcc756f1a08d9e25abe1733f962/executing-2.1.0-py2.py3-none-any.whl", hash = "sha256:8d63781349375b5ebccc3142f4b30350c0cd9c79f921cde38be2be4637e98eaf", size = 25805, upload-time = "2024-09-01T12:37:33.007Z" }, ] +[[package]] +name = "fastexcel" +version = "0.19.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/c8/3b09911348e9c64dbf41096d3e8f0e93c141a23990ec9f32514111bd5f55/fastexcel-0.19.0.tar.gz", hash = "sha256:216c3719ee90963bd93a0bf8c10b177233046ac975b67651152fdaedd3c99aa1", size = 60323, upload-time = "2026-01-20T11:17:37.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/e0/3820e93ea606549cfddb8c437141dd69f2b245e74785efc8bd7511ba909d/fastexcel-0.19.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:68601072a0b4b4277c165b68f1055f88ef7ffe7ed6f08c1eeda0f0271e3f7da0", size = 3082362, upload-time = "2026-01-20T11:17:27.157Z" }, + { url = "https://files.pythonhosted.org/packages/66/0f/b42dc09515879192919942157292912393584045fd8bad98bd92961d4c30/fastexcel-0.19.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c8a87d94445678e7e3f46a6aa39d2afaee5b88a983ec3661143a6488d8955f44", size = 2864365, upload-time = "2026-01-20T11:17:28.786Z" }, + { url = "https://files.pythonhosted.org/packages/8e/4a/bc358b20fcff64b4c14ff7d7a0e1f797792b8b77e30ae755873c02362538/fastexcel-0.19.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e94fc1be6642555f277af792c22a9f80ec9b4d640d9690f00abb822b6d865069", size = 3186426, upload-time = "2026-01-20T11:17:19.087Z" }, + { url = "https://files.pythonhosted.org/packages/58/ae/d2ffdc5ad14190153e2422fc90a1052a4b0c3086d24cb8ae8967575321d8/fastexcel-0.19.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:334f9f40cd68b5924a712b6c104949757a0b8ad8a7e3fa3f3fad1c1ebc00258b", size = 3365628, upload-time = "2026-01-20T11:17:21.116Z" }, + { url = "https://files.pythonhosted.org/packages/6e/67/5f6d4e7760dc3dd8244cd124dabdd5bb7622bf1197edcc2513648847690e/fastexcel-0.19.0-cp310-abi3-win_amd64.whl", hash = "sha256:fbbdf9de79c3ef3572809bb187927c0dc5840968ffe513ea015a383024b7c6b0", size = 2905173, upload-time = "2026-01-20T11:17:33.687Z" }, +] + [[package]] name = "fastjsonschema" version = "2.21.1" @@ -1725,10 +1738,11 @@ wheels = [ [[package]] name = "python4ds" -version = "1.0.4" +version = "0.0.1" source = { virtual = "." } dependencies = [ { name = "beautifulsoup4" }, + { name = "fastexcel" }, { name = "graphviz" }, { name = "ibis-framework", extra = ["sqlite"] }, { name = "ipykernel" }, @@ -1759,12 +1773,14 @@ dependencies = [ { name = "toml" }, { name = "watermark" }, { name = "wbgapi" }, + { name = "xlsxwriter" }, { name = "yfinance" }, ] [package.metadata] requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.12.3" }, + { name = "fastexcel", specifier = ">=0.19.0" }, { name = "graphviz", specifier = ">=0.20.3" }, { name = "ibis-framework", extras = ["sqlite"], specifier = ">=9.5.0" }, { name = "ipykernel", specifier = ">=6.29.5" }, @@ -1795,6 +1811,7 @@ requires-dist = [ { name = "toml", specifier = ">=0.10.2" }, { name = "watermark", specifier = ">=2.5.0" }, { name = "wbgapi", specifier = ">=1.0.14" }, + { name = "xlsxwriter", specifier = ">=3.2.0" }, { name = "yfinance", specifier = ">=1.2.1" }, ] @@ -2463,6 +2480,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/21/02/88b65cc394961a60c43c70517066b6b679738caf78506a5da7b88ffcb643/widgetsnbextension-4.0.13-py3-none-any.whl", hash = "sha256:74b2692e8500525cc38c2b877236ba51d34541e6385eeed5aec15a70f88a6c71", size = 2335872, upload-time = "2024-08-22T12:18:19.491Z" }, ] +[[package]] +name = "xlsxwriter" +version = "3.2.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/2c/c06ef49dc36e7954e55b802a8b231770d286a9758b3d936bd1e04ce5ba88/xlsxwriter-3.2.9.tar.gz", hash = "sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c", size = 215940, upload-time = "2025-09-16T00:16:21.63Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/0c/3662f4a66880196a590b202f0db82d919dd2f89e99a27fadef91c4a33d41/xlsxwriter-3.2.9-py3-none-any.whl", hash = "sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3", size = 175315, upload-time = "2025-09-16T00:16:20.108Z" }, +] + [[package]] name = "yfinance" version = "1.2.1" diff --git a/vis-layers.ipynb b/vis-layers.ipynb index ba22bb2..cf6a92d 100644 --- a/vis-layers.ipynb +++ b/vis-layers.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "95f0a171", + "id": "0", "metadata": {}, "source": [ "# Layers {#sec-vis-layers}\n" @@ -10,7 +10,7 @@ }, { "cell_type": "markdown", - "id": "9fdd3b8a", + "id": "1", "metadata": {}, "source": [ "## Introduction\n", @@ -30,12 +30,12 @@ }, { "cell_type": "markdown", - "id": "17575f3a", + "id": "2", "metadata": {}, "source": [ "### Prerequisites\n", "\n", - "You will need to install the **letsplot** package for this chapter, as well as **pandas**.\n", + "You will need to install the **letsplot** package for this chapter, as well as **polars**.\n", "\n", "In your Python session, import the libraries we'll be using:" ] @@ -43,11 +43,11 @@ { "cell_type": "code", "execution_count": null, - "id": "a86fb211", + "id": "3", "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import polars as pl\n", "from lets_plot import *\n", "\n", "LetsPlot.setup_html()" @@ -55,7 +55,7 @@ }, { "cell_type": "markdown", - "id": "55b00fde", + "id": "4", "metadata": {}, "source": [ "## Aesthetic mappings\n", @@ -68,35 +68,29 @@ { "cell_type": "code", "execution_count": null, - "id": "39a6d993", + "id": "5", "metadata": {}, "outputs": [], "source": [ - "mpg = pd.read_csv(\n", - " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/mpg.csv\", index_col=0\n", + "mpg = pl.read_csv(\"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/mpg.csv\")\n", + "\n", + "mpg = mpg.with_columns(\n", + " [\n", + " pl.col(\"manufacturer\").cast(pl.Categorical),\n", + " pl.col(\"model\").cast(pl.Categorical),\n", + " pl.col(\"trans\").cast(pl.Categorical),\n", + " pl.col(\"drv\").cast(pl.Categorical),\n", + " pl.col(\"fl\").cast(pl.Categorical),\n", + " pl.col(\"class\").cast(pl.Categorical),\n", + " ]\n", ")\n", "\n", - "mpg = mpg.astype(\n", - " {\n", - " \"manufacturer\": \"category\",\n", - " \"model\": \"category\",\n", - " \"displ\": \"double\",\n", - " \"year\": \"int64\",\n", - " \"cyl\": \"int64\",\n", - " \"trans\": \"category\",\n", - " \"drv\": \"category\",\n", - " \"cty\": \"double\",\n", - " \"hwy\": \"double\",\n", - " \"fl\": \"category\",\n", - " \"class\": \"category\",\n", - " }\n", - ")\n", "mpg.head()" ] }, { "cell_type": "markdown", - "id": "6d6f1307", + "id": "6", "metadata": {}, "source": [ "Among the variables in `mpg` are:\n", @@ -118,7 +112,7 @@ { "cell_type": "code", "execution_count": null, - "id": "fe77349a", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -128,7 +122,7 @@ { "cell_type": "code", "execution_count": null, - "id": "e77b5640", + "id": "8", "metadata": {}, "outputs": [], "source": [ @@ -137,7 +131,7 @@ }, { "cell_type": "markdown", - "id": "53e51510", + "id": "9", "metadata": {}, "source": [ "Similarly, we can map `class` to `size` or `alpha` aesthetics as well, which control the shape and the transparency of the points, respectively." @@ -146,7 +140,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ef221330", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -156,7 +150,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d042255e", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -165,7 +159,7 @@ }, { "cell_type": "markdown", - "id": "ec07349f", + "id": "12", "metadata": {}, "source": [ "While we are able to do it, mapping an unordered discrete (categorical) variable (`class`) to an ordered aesthetic variable (`size` or `alpha`) is generally not a good idea because it implies a ranking that does not in fact exist.\n", @@ -182,7 +176,7 @@ { "cell_type": "code", "execution_count": null, - "id": "618edcb4", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -191,7 +185,7 @@ }, { "cell_type": "markdown", - "id": "52611640", + "id": "14", "metadata": {}, "source": [ "Here, the colour doesn't convey information about a variable, but only changes the appearance of the plot.\n", @@ -206,7 +200,7 @@ }, { "cell_type": "markdown", - "id": "37d49ff8", + "id": "15", "metadata": {}, "source": [ "So far we have discussed aesthetics that we can map or set in a scatterplot, when using a point geom.\n", @@ -217,7 +211,7 @@ }, { "cell_type": "markdown", - "id": "3f1da019", + "id": "16", "metadata": {}, "source": [ "1. Create a scatterplot of `hwy` vs. `displ` where the points are pink filled in triangles.\n", @@ -240,7 +234,7 @@ }, { "cell_type": "markdown", - "id": "83aa98f0", + "id": "17", "metadata": {}, "source": [ "## Geometric objects\n", @@ -251,7 +245,7 @@ { "cell_type": "code", "execution_count": null, - "id": "277a4c0f", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -261,7 +255,7 @@ { "cell_type": "code", "execution_count": null, - "id": "07247ba9", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -270,7 +264,7 @@ }, { "cell_type": "markdown", - "id": "58824a10", + "id": "20", "metadata": {}, "source": [ "Both plots contain the same x variable, the same y variable, and both describe the same data.\n", @@ -283,7 +277,7 @@ }, { "cell_type": "markdown", - "id": "0db26c6c", + "id": "21", "metadata": {}, "source": [ "Every geom function in **lets-plot** takes a `mapping` argument, either defined locally in the geom layer or globally in the `ggplot()` layer.\n", @@ -299,7 +293,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4b20c825", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -309,7 +303,7 @@ { "cell_type": "code", "execution_count": null, - "id": "84df3e78", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -318,7 +312,7 @@ }, { "cell_type": "markdown", - "id": "7b114911", + "id": "24", "metadata": {}, "source": [ "Here, `geom_smooth()` separates the cars into three lines based on their `drv` value, which describes a car's drive train.\n", @@ -331,7 +325,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c9e8d92f", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -344,7 +338,7 @@ }, { "cell_type": "markdown", - "id": "b6392da1", + "id": "26", "metadata": {}, "source": [ "Notice that this plot contains two geoms in the same graph.\n", @@ -363,7 +357,7 @@ { "cell_type": "code", "execution_count": null, - "id": "b3916558", + "id": "27", "metadata": {}, "outputs": [], "source": [ @@ -372,7 +366,7 @@ }, { "cell_type": "markdown", - "id": "88708546", + "id": "28", "metadata": {}, "source": [ "You can use the same idea to specify different data for each layer.\n", @@ -383,23 +377,23 @@ { "cell_type": "code", "execution_count": null, - "id": "38870eb5", + "id": "29", "metadata": {}, "outputs": [], "source": [ "(\n", " ggplot(mpg, aes(x=\"displ\", y=\"hwy\"))\n", " + geom_point()\n", - " + geom_point(data=mpg.loc[mpg[\"class\"] == \"2seater\", :], color=\"red\", size=2)\n", + " + geom_point(data=mpg.filter(pl.col(\"class\") == \"2seater\"), color=\"red\", size=2)\n", " + geom_point(\n", - " data=mpg.loc[mpg[\"class\"] == \"2seater\", :], shape=1, size=3, color=\"red\"\n", + " data=mpg.filter(pl.col(\"class\") == \"2seater\"), shape=1, size=3, color=\"red\"\n", " )\n", ")" ] }, { "cell_type": "markdown", - "id": "35a3d017", + "id": "30", "metadata": {}, "source": [ "Geoms are the fundamental building blocks of **lets-plot**.\n", @@ -415,7 +409,7 @@ }, { "cell_type": "markdown", - "id": "39a12f36", + "id": "31", "metadata": {}, "source": [ "### Exercises\n", @@ -443,7 +437,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ae75c5c1", + "id": "32", "metadata": { "tags": [ "remove-cell" @@ -460,7 +454,7 @@ }, { "cell_type": "markdown", - "id": "4c1d45ab", + "id": "33", "metadata": {}, "source": [ "## Facets\n", @@ -471,7 +465,7 @@ { "cell_type": "code", "execution_count": null, - "id": "cb651300", + "id": "34", "metadata": {}, "outputs": [], "source": [ @@ -480,7 +474,7 @@ }, { "cell_type": "markdown", - "id": "0cd9ef67", + "id": "35", "metadata": {}, "source": [ "To facet your plot with the combination of two variables, switch from `facet_wrap()` to `facet_grid()`." @@ -489,7 +483,7 @@ { "cell_type": "code", "execution_count": null, - "id": "61481052", + "id": "36", "metadata": {}, "outputs": [], "source": [ @@ -498,7 +492,7 @@ }, { "cell_type": "markdown", - "id": "4f502141", + "id": "37", "metadata": {}, "source": [ "By default each of the facets share the same scale and range for x and y axes.\n", @@ -509,7 +503,7 @@ { "cell_type": "code", "execution_count": null, - "id": "adcd9079", + "id": "38", "metadata": {}, "outputs": [], "source": [ @@ -523,7 +517,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ceb2a354", + "id": "39", "metadata": {}, "outputs": [], "source": [ @@ -532,7 +526,7 @@ }, { "cell_type": "markdown", - "id": "5e3a949f", + "id": "40", "metadata": {}, "source": [ "### Exercises\n", @@ -602,7 +596,7 @@ }, { "cell_type": "markdown", - "id": "cacf1fb5", + "id": "41", "metadata": {}, "source": [ "## Statistical transformations\n", @@ -616,34 +610,34 @@ { "cell_type": "code", "execution_count": null, - "id": "f379e31b", + "id": "42", "metadata": {}, "outputs": [], "source": [ - "diamonds = pd.read_csv(\n", - " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/diamonds.csv\",\n", - " index_col=0,\n", + "diamonds = pl.read_csv(\n", + " \"https://vincentarelbundock.github.io/Rdatasets/csv/ggplot2/diamonds.csv\"\n", ")\n", "diamonds_cut_order = [\"Fair\", \"Good\", \"Very Good\", \"Premium\", \"Ideal\"]\n", - "diamonds[\"cut\"] = diamonds[\"cut\"].astype(\n", - " pd.CategoricalDtype(categories=diamonds_cut_order, ordered=True)\n", - ")\n", + "diamonds = diamonds.with_columns(pl.col(\"cut\").cast(pl.Enum(diamonds_cut_order)))\n", + "\n", "diamonds.head()" ] }, { "cell_type": "code", "execution_count": null, - "id": "d8faf1ab", + "id": "43", "metadata": {}, "outputs": [], "source": [ - "(ggplot(diamonds, aes(x=\"cut\")) + geom_bar())" + "diamonds_sorted = diamonds.sort(\"cut\")\n", + "\n", + "(ggplot(diamonds_sorted, aes(x=\"cut\")) + geom_bar())" ] }, { "cell_type": "markdown", - "id": "a27666cf", + "id": "44", "metadata": {}, "source": [ "On the x-axis, the chart displays `cut`, a variable from `diamonds`.\n", @@ -666,7 +660,7 @@ }, { "cell_type": "markdown", - "id": "62519f73", + "id": "45", "metadata": {}, "source": [ "You can learn which stat a geom uses by inspecting the default value for the `stat` argument.\n", @@ -680,22 +674,18 @@ { "cell_type": "code", "execution_count": null, - "id": "ca772dd5", + "id": "46", "metadata": {}, "outputs": [], "source": [ - "(\n", - " ggplot(\n", - " diamonds.value_counts(\"cut\").reset_index(name=\"counts\"),\n", - " aes(x=\"cut\", y=\"counts\"),\n", - " )\n", - " + geom_bar(stat=\"identity\")\n", - ")" + "diamonds_counts = diamonds.group_by(\"cut\").agg(pl.len().alias(\"counts\")).sort(\"cut\")\n", + "\n", + "(ggplot(diamonds_counts, aes(x=\"cut\", y=\"counts\")) + geom_bar(stat=\"identity\"))" ] }, { "cell_type": "markdown", - "id": "e365aaaf", + "id": "47", "metadata": {}, "source": [ "## Position adjustments\n", @@ -707,7 +697,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f8da7d91", + "id": "48", "metadata": {}, "outputs": [], "source": [ @@ -717,7 +707,7 @@ { "cell_type": "code", "execution_count": null, - "id": "088e7550", + "id": "49", "metadata": {}, "outputs": [], "source": [ @@ -726,7 +716,7 @@ }, { "cell_type": "markdown", - "id": "c3f7a0a3", + "id": "50", "metadata": {}, "source": [ "Note what happens if you map the fill aesthetic to another variable, like `class`: the bars are automatically stacked.\n", @@ -736,7 +726,7 @@ { "cell_type": "code", "execution_count": null, - "id": "181c70d2", + "id": "51", "metadata": {}, "outputs": [], "source": [ @@ -745,7 +735,7 @@ }, { "cell_type": "markdown", - "id": "b3f74621", + "id": "52", "metadata": {}, "source": [ "The stacking is performed automatically using the **position adjustment** specified by the `position` argument.\n", @@ -759,7 +749,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a8e9c378", + "id": "53", "metadata": {}, "outputs": [], "source": [ @@ -768,7 +758,7 @@ }, { "cell_type": "markdown", - "id": "2aeffbf7", + "id": "54", "metadata": {}, "source": [ "The identity position adjustment is more useful for 2d geoms, like points, where it is the default.\n", @@ -780,7 +770,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14205000", + "id": "55", "metadata": {}, "outputs": [], "source": [ @@ -789,7 +779,7 @@ }, { "cell_type": "markdown", - "id": "3a8e72bd", + "id": "56", "metadata": {}, "source": [ "- `position = \"dodge\"` places overlapping objects directly *beside* one another.\n", @@ -799,7 +789,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c33c4a03", + "id": "57", "metadata": {}, "outputs": [], "source": [ @@ -808,7 +798,7 @@ }, { "cell_type": "markdown", - "id": "cd46f22b", + "id": "58", "metadata": {}, "source": [ "There's one other type of adjustment that's not useful for bar charts, but can be very useful for scatterplots.\n", @@ -819,7 +809,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ba4161de", + "id": "59", "metadata": {}, "outputs": [], "source": [ @@ -828,7 +818,7 @@ }, { "cell_type": "markdown", - "id": "28a3bfc7", + "id": "60", "metadata": {}, "source": [ "The underlying values of `hwy` and `displ` are rounded so the points appear on a grid and many points overlap each other.\n", @@ -844,7 +834,7 @@ { "cell_type": "code", "execution_count": null, - "id": "414ce7af", + "id": "61", "metadata": {}, "outputs": [], "source": [ @@ -853,7 +843,7 @@ }, { "cell_type": "markdown", - "id": "96277926", + "id": "62", "metadata": {}, "source": [ "Adding randomness seems like a strange way to improve your plot, but while it makes your graph less accurate at small scales, it makes your graph *more* revealing at large scales.\n", @@ -867,7 +857,7 @@ }, { "cell_type": "markdown", - "id": "90d09736", + "id": "63", "metadata": {}, "source": [ "### Exercises\n", @@ -879,7 +869,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9bc38aef", + "id": "64", "metadata": {}, "outputs": [], "source": [ @@ -888,7 +878,7 @@ }, { "cell_type": "markdown", - "id": "57b6523f", + "id": "65", "metadata": {}, "source": [ "2. What, if anything, is the difference between the two plots?\n", @@ -908,7 +898,7 @@ }, { "cell_type": "markdown", - "id": "a3138f75", + "id": "66", "metadata": {}, "source": [ "3. What parameters to `geom_jitter()` control the amount of jittering?\n", @@ -919,7 +909,7 @@ }, { "cell_type": "markdown", - "id": "7d302c31", + "id": "67", "metadata": {}, "source": [ "## The layered grammar of graphics\n", @@ -954,7 +944,7 @@ }, { "cell_type": "markdown", - "id": "c0f7a7ff", + "id": "68", "metadata": {}, "source": [ "At this point, you would have a complete graph, but you could further adjust the positions of the geoms within the coordinate system (a position adjustment) or split the graph into subplots (faceting).\n", @@ -971,9 +961,6 @@ } ], "metadata": { - "interpreter": { - "hash": "9d7534ecd9fbc7d385378f8400cf4d6cb9c6175408a574f1c99c5269f08771cc" - }, "jupytext": { "cell_metadata_filter": "-all", "encoding": "# -*- coding: utf-8 -*-", @@ -981,7 +968,7 @@ "main_language": "python" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": ".venv", "language": "python", "name": "python3" }, @@ -995,7 +982,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/visualise.quarto_ipynb_1 b/visualise.quarto_ipynb_1 new file mode 100644 index 0000000..2ef6d2f --- /dev/null +++ b/visualise.quarto_ipynb_1 @@ -0,0 +1,136 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Visualisation {#sec-visualise}\n", + "\n", + "After reading the first part of the book, you understand the basics of the most important tools for doing data science. Now it’s time to start diving into the details. In this part of the book, you’ll learn about visualising data in further depth (in @sec-vis-layers), and get further stuck into the details of the different kinds of data visualisation (in @sec-exploratory-data-analysis and @sec-communicate-plots). In this short chapter, we discuss the different ways to create visualisations, and the different purposes of visualisations.\n", + "\n", + "## Philosophies of data visualisation\n", + "\n", + "There are broadly two categories of approach to using code to create data visualisations: *imperative* (build what you want from individual elements) and *declarative* (say what you want from a list of pre-existing options). Choosing which to use involves a trade-off: imperative libraries offer you flexibility but at the cost of some verbosity; declarative libraries offer you a quick way to plot your data, but only if it’s in the right format to begin with, and customisation to special chart types is more difficult.\n", + "\n", + "Python has many excellent plotting packages, including perhaps the most powerful imperative plotting package around, **matplotlib**, and an amazing declarative library that we already saw, **lets-plot**. These two libraries will get you a long way, and each could be worthy of an entire book themselves. Fortunately for us, though, we can do 95% of what we need with a small number of commands from one or the other of them. In general, to keep this book as light as possible, we've opted to use **lets-plot** wherever possible—and @sec-vis-layers is going to take you on a more in-depth tour of how to use it yourself.\n", + "\n", + "## Purposes of data visualisation\n", + "\n", + "Data visualisation has all kinds of different purposes. It can be useful to bear in mind three broad categories of visualisation that are out there:\n", + "\n", + "- exploratory\n", + "- scientific\n", + "- narrative\n", + "\n", + "Let's look at each in a bit more detail.\n", + "\n", + "### Exploratory Data Viz\n", + "\n", + "The first of the three kinds is *exploratory data visualisation*, and it's the kind that you do when you're looking and data and trying to understand it. Just plotting the data is a really good strategy for getting a feel for any issues there might be. This is perhaps most famously demonstrated by Anscombe's quartet: four different datasets with the same mean, standard deviation, and correlation but very different data distributions." + ], + "id": "f3331573" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "#| echo: false\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib_inline.backend_inline\n", + "\n", + "# Plot settings\n", + "plt.style.use(\"https://github.com/aeturrell/python4DS/raw/main/plot_style.txt\")\n", + "matplotlib_inline.backend_inline.set_matplotlib_formats(\"svg\")\n", + "\n", + "# Set max rows displayed for readability\n", + "pd.set_option(\"display.max_rows\", 6)\n", + "\n", + "x = [10, 8, 13, 9, 11, 14, 6, 4, 12, 7, 5]\n", + "y1 = [8.04, 6.95, 7.58, 8.81, 8.33, 9.96, 7.24, 4.26, 10.84, 4.82, 5.68]\n", + "y2 = [9.14, 8.14, 8.74, 8.77, 9.26, 8.10, 6.13, 3.10, 9.13, 7.26, 4.74]\n", + "y3 = [7.46, 6.77, 12.74, 7.11, 7.81, 8.84, 6.08, 5.39, 8.15, 6.42, 5.73]\n", + "x4 = [8, 8, 8, 8, 8, 8, 8, 19, 8, 8, 8]\n", + "y4 = [6.58, 5.76, 7.71, 8.84, 8.47, 7.04, 5.25, 12.50, 5.56, 7.91, 6.89]\n", + "\n", + "datasets = {\"I\": (x, y1), \"II\": (x, y2), \"III\": (x, y3), \"IV\": (x4, y4)}\n", + "\n", + "fig, axs = plt.subplots(\n", + " 2,\n", + " 2,\n", + " sharex=True,\n", + " sharey=True,\n", + " figsize=(10, 6),\n", + " gridspec_kw={\"wspace\": 0.08, \"hspace\": 0.08},\n", + ")\n", + "axs[0, 0].set(xlim=(0, 20), ylim=(2, 14))\n", + "axs[0, 0].set(xticks=(0, 10, 20), yticks=(4, 8, 12))\n", + "\n", + "for ax, (label, (x, y)) in zip(axs.flat, datasets.items()):\n", + " ax.text(0.1, 0.9, label, fontsize=20, transform=ax.transAxes, va=\"top\")\n", + " ax.tick_params(direction=\"in\", top=True, right=True)\n", + " ax.plot(x, y, \"o\")\n", + "\n", + " # linear regression\n", + " p1, p0 = np.polyfit(x, y, deg=1) # slope, intercept\n", + " ax.axline(xy1=(0, p0), slope=p1, color=\"r\", lw=2)\n", + "\n", + " # add text box for the statistics\n", + " stats = (\n", + " f\"$\\\\mu$ = {np.mean(y):.2f}\\n\"\n", + " f\"$\\\\sigma$ = {np.std(y):.2f}\\n\"\n", + " f\"$r$ = {np.corrcoef(x, y)[0][1]:.2f}\"\n", + " )\n", + " bbox = dict(boxstyle=\"round\", fc=\"blanchedalmond\", ec=\"orange\", alpha=0.5)\n", + " ax.text(\n", + " 0.95,\n", + " 0.07,\n", + " stats,\n", + " fontsize=9,\n", + " bbox=bbox,\n", + " transform=ax.transAxes,\n", + " horizontalalignment=\"right\",\n", + " )\n", + "\n", + "plt.suptitle(\"Anscombe's Quartet\")\n", + "plt.show()" + ], + "id": "64a0e7f6", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Exploratory visualisation is usually quick and dirty, and flexible too. Some exploratory data viz can be automated, and there's a whole host of packages to help with this, including [**skimpy**](https://aeturrell.github.io/skimpy/).\n", + "\n", + "Beyond you and perhaps your co-authors/collaborators, however, not many other people should be seeing your exploratory visualisation! They will typically be worked up quickly, be numerous, and be throw-away. We'll look more at this in @sec-exploratory-data-analysis.\n", + "\n", + "### Scientific Data Viz\n", + "\n", + "The second kind, scientific data visualisation, is the prime cut of your exploratory visualisation. It's the kind of plot you might include in a more technical paper, the picture that says a thousand words. I often think of the first image of a black hole @akiyama2019first as a prime example of this. You can get away with having a high density of information in a scientific plot and, in short format journals, you may need to. The journal Physical Review Letters, which has an 8 page limit, has a classic of this genre in more or less every issue. Ensuring that important values can be accurately read from the plot is especially important in these kinds of charts. But they can also be the kind of plot that presents the killer results in a study; they might not be exciting to people who don't look at charts for a living, but they might be exciting and, just as importantly, understandable by your peers.\n", + "\n", + "This type of visualisation is especially popular in the big science journals like *Nature* and *Science*, where space is at a premium. We won't cover this type of plot in this book, because it tends to be very bespoke.\n", + "\n", + "### Narrative Data Viz\n", + "\n", + "The third and final kind is narrative data visualisation. This is the one that requires the most thought in the step where you go from the first view to the end product. It's a visualisation that doesn't just show a picture, but gives an insight. These are the kind of visualisations that you might see in the *Financial Times*, *The Economist*, or on the *BBC News* website. They come with aids that help the viewer focus on the aspects that the creator wanted them to (you can think of these aids or focuses as doing for visualisation what bold font does for text). They're well worth using in your work, especially if you're trying to communicate a particular narrative, and especially if the people you're communicating with don't have deep knowledge of the topic. You might use them in a paper that you hope will have a wide readership, in a blog post summarising your work, or in a report intended for a policymaker.\n", + "\n", + "You can find more information on the topic of communicating via data visualisations in the @sec-communicate-plots chapter." + ], + "id": "30b9ff30" + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "language": "python", + "display_name": "Python 3 (ipykernel)", + "path": "/Users/omagic/Documents/GitHub/python4DSpolars/.venv/share/jupyter/kernels/python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/webscraping-and-apis.ipynb b/webscraping-and-apis.ipynb index 9366eb6..802151d 100644 --- a/webscraping-and-apis.ipynb +++ b/webscraping-and-apis.ipynb @@ -10,7 +10,7 @@ "\n", "## Introduction\n", "\n", - "This chapter will show you how to work with online data that is either obtained from webpages via webscraping or more directly over the internet via an API. An important principle is always to use an API if one is available as this is designed to pass information directly into your Python session and will save you a lot of effort." + "This chapter will show you how to work with online data that is either obtained from webpages via webscraping or more directly over the internet via an API. An important principle is always to use an API if one is available as this is designed to pass information directly into your Python session and will save you a lot of effort.\n" ] }, { @@ -56,7 +56,7 @@ "\n", "As a brief example, in the US, lists of ingredients and instructions are not copyrightable, so copyright can not be used to protect a recipe. But if that list of recipes is accompanied by substantial novel literary content, that is copyrightable. This is why when you’re looking for a recipe on the internet there’s always so much content beforehand.\n", "\n", - "If you do need to scrape original content (like text or images), you may still be protected under the doctrine of fair use. Fair use is not a hard and fast rule, but weighs up a number of factors. It’s more likely to apply if you are collecting the data for research or non-commercial purposes and if you limit what you scrape to just what you need." + "If you do need to scrape original content (like text or images), you may still be protected under the doctrine of fair use. Fair use is not a hard and fast rule, but weighs up a number of factors. It’s more likely to apply if you are collecting the data for research or non-commercial purposes and if you limit what you scrape to just what you need.\n" ] }, { @@ -67,9 +67,9 @@ "source": [ "### Prerequisites\n", "\n", - "You will need to install the **pandas** package for this chapter. We'll use **seaborn** too, which you should already have installed. You will also need to install the **beautifulsoup**, **pandas-datareader**, and **wbgapi** packages in your terminal using `uv add beautifulsoup4`, `uv add pandas-datareader`, and `uv add wbgapi` respectively. We'll also use two built-in packages, **textwrap** and **requests**.\n", + "You will need to install the **pandas** and **polars** package for this chapter. We'll use **seaborn** too, which you should already have installed. You will also need to install the **beautifulsoup**, **pandas-datareader**, and **wbgapi** packages in your terminal using `uv add beautifulsoup4`, and `uv add wbgapi` respectively. We'll also use two built-in packages, **textwrap** and **requests**.\n", "\n", - "To kick off, let's import some of the packages we need (it's always good practice to import the packages you need at the top of a script or notebook)." + "To kick off, let's import some of the packages we need (it's always good practice to import the packages you need at the top of a script or notebook).\n" ] }, { @@ -81,12 +81,11 @@ "source": [ "import textwrap\n", "\n", + "import lets_plot as lp\n", "import pandas as pd\n", + "import polars as pl\n", "import requests\n", - "from bs4 import BeautifulSoup\n", - "from lets_plot import *\n", - "\n", - "LetsPlot.setup_html()" + "from bs4 import BeautifulSoup" ] }, { @@ -95,9 +94,9 @@ "id": "f43a5237", "metadata": {}, "source": [ - "## Extracting Data from Files on the Internet using **pandas**\n", + "## Extracting Data from Files on the Internet using **polars**\n", "\n", - "It's easy to read data from the internet once you have the url and file type. Here, for instance, is an example that reads in the 'storms' dataset, which is stored as a CSV file in a URL (we'll only grab the first 10 rows):" + "It's easy to read data from the internet once you have the url and file type. Here, for instance, is an example that reads in the 'storms' dataset, which is stored as a CSV file in a URL (we'll only grab the first 10 rows):\n" ] }, { @@ -107,8 +106,8 @@ "metadata": {}, "outputs": [], "source": [ - "pd.read_csv(\n", - " \"https://vincentarelbundock.github.io/Rdatasets/csv/dplyr/storms.csv\", nrows=10\n", + "pl.read_csv(\n", + " \"https://vincentarelbundock.github.io/Rdatasets/csv/dplyr/storms.csv\", n_rows=10\n", ")" ] }, @@ -122,7 +121,7 @@ "\n", "Using an API (application programming interface) is another way to draw down information from the interweb. Their just a way for one tool, say Python, to speak to another tool, say a server, and usefully exchange information. The classic use case would be to post a request for data that fits a certain query via an API and to get a download of that data back in return. (You should always preferentially use an API over webscraping a site.)\n", "\n", - "Because they are designed to work with any tool, you don't actually need a programming language to interact with an API, it's just a *lot* easier if you do.\n", + "Because they are designed to work with any tool, you don't actually need a programming language to interact with an API, it's just a _lot_ easier if you do.\n", "\n", "::: {.callout-note}\n", "An API key is needed in order to access some APIs. Sometimes all you need to do is register with site, in other cases you may have to pay for access.\n", @@ -132,13 +131,13 @@ "\n", "An API has an 'endpoint', the base url, and then a URL that encodes the question. Let's see an example with the ONS API for which the endpoint is \"https://api.beta.ons.gov.uk/v1/\". The rest of the API has the form 'data?uri=' and then the long ID of both the timeseries (jp9z) and then the dataset (LMS), which is vacancies in the UK services sector.\n", "\n", - "The data that are returned by APIs are typically in JSON format, which looks a lot like a nested Python dictionary and its entries can be accessed in the same way--this is what is happening when getting the series' title in the example below. JSON is not good for analysis, so we'll use **pandas** to put the data into shape." + "The data that are returned by APIs are typically in JSON format, which looks a lot like a nested Python dictionary and its entries can be accessed in the same way--this is what is happening when getting the series' title in the example below. JSON is not good for analysis, so we'll use **polars** to put the data into shape.\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "c4226d67", + "id": "6107093c", "metadata": {}, "outputs": [], "source": [ @@ -147,18 +146,36 @@ "# Get the data from the ONS API:\n", "json_data = requests.get(url).json()\n", "\n", - "# Prep the data for a quick plot\n", "title = json_data[\"description\"][\"title\"]\n", + "\n", + "# Convert dates using string operations\n", "df = (\n", - " pd.DataFrame(pd.json_normalize(json_data[\"months\"]))\n", - " .assign(\n", - " date=lambda x: pd.to_datetime(x[\"date\"]),\n", - " value=lambda x: pd.to_numeric(x[\"value\"]),\n", + " pl.DataFrame(json_data[\"months\"])\n", + " .with_columns(\n", + " [\n", + " # Add day to make it a valid date string\n", + " (pl.col(\"date\") + \"-01\").str.to_date(format=\"%Y %b-%d\").alias(\"date\"),\n", + " pl.col(\"value\").cast(pl.Float64).alias(\"value\"),\n", + " ]\n", " )\n", - " .set_index(\"date\")\n", + " .drop_nulls(\"date\")\n", + " .sort(\"date\")\n", + ")\n", + "\n", + "\n", + "# Initialize the library\n", + "lp.LetsPlot.setup_html()\n", + "\n", + "# Create plot using the alias\n", + "chart = (\n", + " lp.ggplot(df, lp.aes(x=\"date\", y=\"value\"))\n", + " + lp.geom_line(size=2.0, color=\"steelblue\")\n", + " + lp.ggtitle(title)\n", + " + lp.ylim(0, df[\"value\"].max() * 1.2)\n", + " + lp.theme_classic()\n", ")\n", "\n", - "df[\"value\"].plot(title=title, ylim=(0, df[\"value\"].max() * 1.2), lw=3.0);" + "chart" ] }, { @@ -167,37 +184,9 @@ "id": "670ce0bb", "metadata": {}, "source": [ - "We've talked about *reading* APIs. You can also create your own to serve up data, models, whatever you like! This is an advanced topic and we won't cover it; but if you do need to, the simplest way is to use [Fast API](https://fastapi.tiangolo.com/). You can find some short video tutorials for Fast API [here](https://calmcode.io/fastapi/hello-world.html).\n", - "\n", - "### Pandas Datareader: an easier way to interact with (some) APIs\n", - "\n", - "Although it didn't take much code to get the ONS data, it would be even better if it was just a single line, wouldn't it? Fortunately there are some packages out there that make this easy, but it does depend on the API (and APIs come and go over time).\n", - "\n", - "By far the most comprehensive library for accessing extra APIs is [**pandas-datareader**](https://pandas-datareader.readthedocs.io/en/latest/), which provides convenient access to:\n", - "\n", - "- FRED\n", - "- Quandl\n", - "- World Bank\n", - "- OECD\n", - "- Eurostat\n", + "We've talked about _reading_ APIs. You can also create your own to serve up data, models, whatever you like! This is an advanced topic and we won't cover it; but if you do need to, the simplest way is to use [Fast API](https://fastapi.tiangolo.com/). You can find some short video tutorials for Fast API [here](https://calmcode.io/fastapi/hello-world.html).\n", "\n", - "and more.\n", - "\n", - "Let's see an example using FRED (the Federal Reserve Bank of St. Louis' economic data library). This time, let's look at the UK unemployment rate:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bf758fb4", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas_datareader.data as web\n", - "\n", - "df_u = web.DataReader(\"LRHUTTTTGBM156S\", \"fred\")\n", - "\n", - "df_u.plot(title=\"UK unemployment (percent)\", legend=False, ylim=(2, 6), lw=3.0);" + "### Accessing World Bank Data with wbgapi\n" ] }, { @@ -206,7 +195,9 @@ "id": "0613aefb", "metadata": {}, "source": [ - "And, because it's also a really useful one, let's see how to use the [**wbgapi**](https://pypi.org/project/wbgapi/) package to access World Bank data. (**pandas-datareader** used to provide a World Bank reader too, but it has not been actively maintained, so we prefer **wbgapi** for new work.)" + "While APIs can be accessed directly using tools like requests, some specialized libraries make working with structured datasets much easier. One such example is wbgapi, which provides a convenient interface for accessing World Bank data.\n", + "\n", + "Let’s look at an example using World Bank data on CO₂-equivalent emissions per capita:\n" ] }, { @@ -224,18 +215,21 @@ "import wbgapi as wb\n", "\n", "indicator_code = \"EN.GHG.ALL.PC.CE.AR5\"\n", + "\n", "df = (\n", - " wb.data.DataFrame(\n", - " indicator_code,\n", - " [\"USA\", \"CHN\", \"IND\", \"EAS\", \"ECS\"], # country and region codes\n", - " time=range(2019, 2020),\n", - " labels=True,\n", + " pl.from_pandas(\n", + " wb.data.DataFrame(\n", + " indicator_code,\n", + " [\"USA\", \"CHN\", \"IND\", \"EAS\", \"ECS\"],\n", + " time=range(2019, 2020),\n", + " labels=True,\n", + " ).reset_index()\n", " )\n", - " .rename(columns={\"Country\": \"country\", \"YR2019\": indicator_code})\n", - " .reset_index(drop=True)\n", + " .rename({\"Country\": \"country\"})\n", + " .with_columns(pl.col(\"country\").map_elements(lambda x: textwrap.fill(x, 10)))\n", + " .sort(indicator_code, descending=True)\n", ")\n", - "df[\"country\"] = df[\"country\"].apply(lambda x: textwrap.fill(x, 10)) # wrap long names\n", - "df = df.sort_values(indicator_code) # re-order\n", + "\n", "df.head()" ] }, @@ -246,19 +240,26 @@ "metadata": {}, "outputs": [], "source": [ - "(\n", - " ggplot(df, aes(x=\"country\", y=indicator_code))\n", - " + geom_bar(aes(fill=\"country\"), color=\"black\", alpha=0.8, stat=\"identity\")\n", - " + scale_fill_discrete()\n", - " + theme_minimal()\n", - " + theme(legend_position=\"none\")\n", - " + ggsize(600, 400)\n", - " + labs(\n", + "lp.LetsPlot.setup_html()\n", + "\n", + "country_order = df[\"country\"].to_list()\n", + "\n", + "plot = (\n", + " lp.ggplot(df, lp.aes(x=\"country\", y=indicator_code))\n", + " + lp.geom_bar(lp.aes(fill=\"country\"), color=\"black\", alpha=0.8, stat=\"identity\")\n", + " + lp.scale_x_discrete(limits=country_order)\n", + " + lp.scale_fill_discrete()\n", + " + lp.theme_minimal()\n", + " + lp.theme(legend_position=\"none\")\n", + " + lp.ggsize(600, 400)\n", + " + lp.labs(\n", " subtitle=\"Greenhouse gases (CO2-equivalent metric tons per capita, 2019)\",\n", " title=\"The USA leads the world on per-capita emissions\",\n", " y=\"\",\n", " )\n", - ")" + ")\n", + "\n", + "plot.show()" ] }, { @@ -267,15 +268,19 @@ "id": "b7bf16d7", "metadata": {}, "source": [ - "### The OECD API\n", + "### The Eurostat SDMX API\n", + "\n", + "Sometimes it’s convenient to use APIs directly. The Eurostat API provides access to a massive repository of European statistical data using the SDMX (Statistical Data and Metadata eXchange) standard. While Eurostat offers multiple formats, using the SDMX-ML (XML) format via the sdmx1 library allows us to pull structured data into the Python ecosystem with high precision.\n", + "\n", + "Key to using the Eurostat API is understanding the Data Structure Definition (DSD). Every dataset is essentially a multidimensional \"cube\" where each dimension (like Geography, Unit, or Frequency) has specific codes.\n", "\n", - "Sometimes it's convenient to use APIs directly, and, as an example, the OECD API comes with a LOT of complexity that direct access can take advantage of. The OECD API makes data available in both JSON and XML formats, and we'll use [**pandasdmx**](https://pandasdmx.readthedocs.io/) (aka the Statistical Data and Metadata eXchange (SDMX) package for the Python data ecosystem) to pull down the XML format data and turn it into a regular **pandas** data frame.\n", + "To find the exact codes you need:\n", "\n", - "Now, key to using the OECD API is knowledge of its many codes: for countries, times, resources, and series. You can find some broad guidance on what codes the API uses [here](https://data.oecd.org/api/sdmx-ml-documentation/) but to find exactly what you need can be a bit tricky. Two tips are:\n", - "1. If you know what you're looking for is in a particular named dataset, eg \"QNA\" (Quarterly National Accounts), put `https://stats.oecd.org/restsdmx/sdmx.ashx/GetDataStructure/QNA/all?format=SDMX-ML` into your browser and look through the XML file; you can pick out the sub-codes and the countries that are available.\n", - "2. Browse around on https://stats.oecd.org/ and use Customise then check all the \"Use Codes\" boxes to see whatever your browsing's code names.\n", + "The Data Browser: Browse the Eurostat Data Navigation Tree. Once you find a table (e.g., \"HICP - monthly data\"), the \"Dataset Code\" (like prc_hicp_manr) is shown in brackets.\n", "\n", - "Let's see an example of this in action. We'd like to see the productivity (GDP per hour) data for a range of countries since 2010. We are going to be in the productivity resource (code \"PDB_LV\") and we want the USD current prices (code \"CPC\") measure of GDP per employed worker (code \"T_GDPEMP) from 2010 onwards (code \"startTime=2010\"). We'll grab this for some developed countries where productivity measurements might be slightly more comparable. The comments below explain what's happening in each step." + "Positional Keys: Eurostat's REST API expects a \"key string\" where codes are placed in a specific order separated by dots (e.g., Freq.Unit.Item.Geo). If you know the order, you can \"slice\" the data cube directly.\n", + "\n", + "Let’s see an example of this in action. We want to see the Harmonised Index of Consumer Prices (HICP)—specifically the annual rate of change for all items—for Germany and France. We will use the resource prc_hicp_manr, requesting Monthly frequency (M), the Annual Rate of Change unit (RCH_A), and the \"All-items\" classification (CP00).\n" ] }, { @@ -285,18 +290,33 @@ "metadata": {}, "source": [ "```python\n", - "import pandasdmx as pdmx\n", - "# Tell pdmx we want OECD data\n", - "oecd = pdmx.Request(\"OECD\")\n", - "# Set out everything about the request in the format specified by the OECD API\n", - "data = oecd.data(\n", - " resource_id=\"PDB_LV\",\n", - " key=\"GBR+FRA+CAN+ITA+DEU+JPN+USA.T_GDPEMP.CPC/all?startTime=2010\",\n", - ").to_pandas()\n", + "import sdmx\n", + "import polars as pl\n", + "\n", + "# Tell sdmx we want ESTAT data\n", + "client = sdmx.Client('ESTAT')\n", + "\n", + "# 2. Build the URL-style positional key\n", + "# Format: [Freq].[Unit].[Coicop].[Geo]\n", + "# We use '+' to join multiple countries (DE and FR)\n", + "resource_id = 'prc_hicp_manr'\n", + "key_string = 'M.RCH_A.CP00.DE+FR'\n", + "\n", + "# 3. Fetch the data directly\n", + "# 'startPeriod' limits the timeline to recent data\n", + "response = client.data(\n", + " resource_id=resource_id,\n", + " key=key_string,\n", + " params={'startPeriod': '2024-01'}\n", + ")\n", "\n", - "df = pd.DataFrame(data).reset_index()\n", - "df.head()\n", - "```" + "# 4. Convert the SDMX-ML response to a Polars DataFrame\n", + "# We bridge through Pandas as sdmx1 is optimized for it\n", + "df_pd = sdmx.to_pandas(response).to_frame(name='value').reset_index()\n", + "df = pl.from_pandas(df_pd)\n", + "\n", + "print(df.head())\n", + "```\n" ] }, { @@ -305,13 +325,13 @@ "id": "e5cac233", "metadata": {}, "source": [ - "| | LOCATION | SUBJECT | MEASURE | TIME_PERIOD | value |\n", - "|--:|---------:|---------:|--------:|------------:|-------------:|\n", - "| 0 | CAN | T_GDPEMP | CPC | 2010 | 78848.604088 |\n", - "| 1 | CAN | T_GDPEMP | CPC | 2011 | 81422.364748 |\n", - "| 2 | CAN | T_GDPEMP | CPC | 2012 | 82663.028058 |\n", - "| 3 | CAN | T_GDPEMP | CPC | 2013 | 86368.582158 |\n", - "| 4 | CAN | T_GDPEMP | CPC | 2014 | 89617.632446 |" + "| | TIME_PERIOD | geo | unit | freq | coicop | value |\n", + "| --: | ----------: | :-- | :---- | :--- | :----- | ----: |\n", + "| 0 | 2024-01 | DE | RCH_A | M | CP00 | 3.1 |\n", + "| 1 | 2024-02 | DE | RCH_A | M | CP00 | 2.7 |\n", + "| 2 | 2024-03 | DE | RCH_A | M | CP00 | 2.3 |\n", + "| 3 | 2024-04 | DE | RCH_A | M | CP00 | 2.4 |\n", + "| 4 | 2024-05 | DE | RCH_A | M | CP00 | 2.8 |\n" ] }, { @@ -320,7 +340,7 @@ "id": "302326b4", "metadata": {}, "source": [ - "Great that worked! We have data in a nice tidy format." + "Great that worked! We have data in a nice tidy format.\n" ] }, { @@ -334,7 +354,7 @@ "- There is a regularly updated list of APIs over at this [public APIs repo on github](https://github.com/public-apis/public-apis). It doesn't have an economics section (yet), but it has a LOT of other APIs.\n", "- Berkeley Library maintains a [list of economics APIs](https://guides.lib.berkeley.edu/c.php?g=4395&p=7995952) that is well worth looking through.\n", "- [NASDAQ Data Link](https://docs.data.nasdaq.com/), which has a great deal of [financial data](https://docs.data.nasdaq.com/docs/data-organization).\n", - "- [DBnomics](https://db.nomics.world/): publicly-available economic data provided by national and international statistical institutions, but also by researchers and private companies." + "- [DBnomics](https://db.nomics.world/): publicly-available economic data provided by national and international statistical institutions, but also by researchers and private companies.\n" ] }, { @@ -347,7 +367,7 @@ "\n", "Webscraping is a way of grabbing information from the internet that was intended to be displayed in a browser. But it should only be used as a last resort, and only then when permitted by the terms and conditions of a website.\n", "\n", - "If you're getting data from the internet, it's much better to use an API whenever you can: grabbing information in a structure way is *exactly* why APIs exist. APIs should also be more stable than websites, which may change frequently. Typically, if an organisation is happy for you to grab their data, they will have made an API expressly for that purpose. It's pretty rare that there's a major website which *does* permit webscraping but which doesn't have an API; for these websites, if they don't have an API, chances scraping is against their terms and conditions. Those terms and conditions may be enforceable by law (different rules in different countries here, and you really need legal advice if it's not unambiguous as to whether you can scrape or not.)\n", + "If you're getting data from the internet, it's much better to use an API whenever you can: grabbing information in a structure way is _exactly_ why APIs exist. APIs should also be more stable than websites, which may change frequently. Typically, if an organisation is happy for you to grab their data, they will have made an API expressly for that purpose. It's pretty rare that there's a major website which _does_ permit webscraping but which doesn't have an API; for these websites, if they don't have an API, chances scraping is against their terms and conditions. Those terms and conditions may be enforceable by law (different rules in different countries here, and you really need legal advice if it's not unambiguous as to whether you can scrape or not.)\n", "\n", "There are other reasons why webscraping is not so good; for example, if you need a back-run then it might be offered through an API but not shown on the webpage. (Or it might not be available at all, in which case it's best to get in touch with the organisation or check out WaybackMachine in case they took snapshots).\n", "\n", @@ -355,13 +375,13 @@ "\n", "If you do find yourself in a scraping situation, be really sure to check that's legally allowed and also that you are not violating the website's `robots.txt` rules: this is a special file on almost every website that sets out what's fair play to crawl (conditional on legality) and what robots should not go poking around in.\n", "\n", - "In Python, you are spoiled for choice when it comes to webscraping. There are five very strong libraries that cover a real range of user styles and needs: **requests**, **lxml**, **beautifulsoup**, **selenium**, and *scrapy**.\n", + "In Python, you are spoiled for choice when it comes to webscraping. There are five very strong libraries that cover a real range of user styles and needs: **requests**, **lxml**, **beautifulsoup**, **selenium**, and \\*scrapy\\*\\*.\n", "\n", "For quick and simple webscraping, my usual combo would **requests**, which does little more than go and grab the HTML of a webpage, and **beautifulsoup**, which then helps you to navigate the structure of the page and pull out what you're actually interested in. For dynamic webpages that use javascript rather than just HTML, you'll need **selenium**. To scale up and hit thousands of webpages in an efficient way, you might try **scrapy**, which can work with the other tools and handle multiple sessions, and all other kinds of bells and whistles... it's actually a \"web scraping framework\".\n", "\n", "It's always helpful to see coding in practice, so that's what we'll do now, but note that we'll be skipping over a lot of important detail such as user agents, being 'polite' with your scraping requests, being efficient with caching and crawling.\n", "\n", - "In lieu of a better example, let's scrape the research page of [http://aeturrell.com/](http://aeturrell.com/)" + "In lieu of a better example, let's scrape the research page of [http://aeturrell.com/](http://aeturrell.com/)\n" ] }, { @@ -384,7 +404,7 @@ "source": [ "Okay, what just happened? We asked requests to grab the HTML of the webpage and then printed the first 300 characters of the text that it found.\n", "\n", - "Let's now parse this into something humans can read (or can read more easily) using beautifulsoup:" + "Let's now parse this into something humans can read (or can read more easily) using beautifulsoup:\n" ] }, { @@ -404,7 +424,7 @@ "id": "5748e928", "metadata": {}, "source": [ - "Now we see more structure of the page and even some *HTML tags* such as 'title' and 'link'. Now we come to the data extraction part: say we want to pull out every paragraph of text, we can use beautifulsoup to skim down the HTML structure and pull out only those parts with the paragraph tag ('p').\n" + "Now we see more structure of the page and even some _HTML tags_ such as 'title' and 'link'. Now we come to the data extraction part: say we want to pull out every paragraph of text, we can use beautifulsoup to skim down the HTML structure and pull out only those parts with the paragraph tag ('p').\n" ] }, { @@ -426,7 +446,7 @@ "id": "2936677e", "metadata": {}, "source": [ - "Although this paragraph isn't too bad, you can make this more readable by stripping out HTML tags altogether with the `.text` method:" + "Although this paragraph isn't too bad, you can make this more readable by stripping out HTML tags altogether with the `.text` method:\n" ] }, { @@ -445,7 +465,7 @@ "id": "9d9d890e", "metadata": {}, "source": [ - "Now let's say we didn't care about most of the page, we *only* wanted to get hold of the names of projects. For this we need to identify the tag type of the element we're interested in, in this case 'div', and it's class type, in this case \"project-name\". We do it like this (and show nice text in the process):\n" + "Now let's say we didn't care about most of the page, we _only_ wanted to get hold of the names of projects. For this we need to identify the tag type of the element we're interested in, in this case 'div', and it's class type, in this case \"project-name\". We do it like this (and show nice text in the process):\n" ] }, { @@ -478,7 +498,7 @@ "info_on_pages = [scraper(root_url + str(i)) for i in range(start, stop)]\n", "```\n", "\n", - "That's all we'll cover here but remember we've barely *scraped* the surface of this big, complex topic. If you want to read about an application, it's hard not to recommend the paper on webscraping that has undoubtedly change the world the most, and very likely has affected your own life in numerous ways: [\"The PageRank Citation Ranking: Bringing Order to the Web\"](http://ilpubs.stanford.edu:8090/422/) by Page, Brin, Motwani and Winograd. For a more in-depth example of webscraping, check out realpython's [tutorial](https://realpython.com/python-web-scraping-practical-introduction/)." + "That's all we'll cover here but remember we've barely _scraped_ the surface of this big, complex topic. If you want to read about an application, it's hard not to recommend the paper on webscraping that has undoubtedly change the world the most, and very likely has affected your own life in numerous ways: [\"The PageRank Citation Ranking: Bringing Order to the Web\"](http://ilpubs.stanford.edu:8090/422/) by Page, Brin, Motwani and Winograd. For a more in-depth example of webscraping, check out realpython's [tutorial](https://realpython.com/python-web-scraping-practical-introduction/).\n" ] }, { @@ -489,11 +509,11 @@ "source": [ "### Webscraping Tables\n", "\n", - "Often there are times when you don't actually want to scrape an entire webpage and all you want is the data from a *table* within the page. Fortunately, there is an easy way to scrape individual tables using the **pandas** package.\n", + "There are times when you don't need to scrape an entire webpage; you simply want the structured data from a specific table. While Polars is a high-performance data engine, it focuses on strict data formats (like Parquet or CSV) and does not natively include an HTML parser. However, we can easily bridge this gap by using Pandas to fetch the table and then converting it into a Polars DataFrame.\n", "\n", - "We will read data from a table on 'https://webscraper.io/test-sites/tables' using **pandas**. The function we'll use is `read_html()`, which returns a list of data frames of all the tables it finds when you pass it a URL. If you want to filter the list of tables, use the `match=` keyword argument with text that only appears in the table(s) you're interested in.\n", + "We will read data from 'https://webscraper.io/test-sites/tables' using `pd.read_html()`. This function scans the webpage and returns a list of all tables it finds as DataFrames. To target a specific table, we use the match= keyword argument with text that uniquely appears in the table we want—in this case, \"First Name\".\n", "\n", - "The example below shows how this works; looking at the website, we can see that the table we're interested in, has a 'First Name' column. Therefore we run:" + "Once captured, we convert the result to Polars using pl.from_pandas() to take advantage of Polars' superior query performance and expression API.\n" ] }, { @@ -503,10 +523,13 @@ "metadata": {}, "outputs": [], "source": [ - "df_list = pd.read_html(\"https://webscraper.io/test-sites/tables\", match=\"First Name\")\n", + "import polars as pl\n", + "\n", + "pd_list = pd.read_html(\"https://webscraper.io/test-sites/tables\", match=\"First Name\")\n", "# Retrieve first entry from list of data frames\n", - "df = df_list[0]\n", - "df.head()" + "df = pl.from_pandas(pd_list[0])\n", + "\n", + "print(df.head())" ] }, { @@ -515,9 +538,9 @@ "id": "31e49317", "metadata": {}, "source": [ - "This gives us the table neatly loaded into a **pandas** data frame ready for further use.\n", + "This gives us the table neatly loaded into a **polars** data frame ready for further use.\n", "\n", - "If you get a '403' error, it means that the website has blocked **pandas** because it can see that you are engaged in web scraping. This is because some people web scrape irresponsibly, or because websites have provided other, preferred ways for you to obtain the data, eg via a download of the whole thing (think Wikipedia) or through an API. (If you really need to, [you can often get around the 403 error](https://stackoverflow.com/questions/43590153/http-error-403-forbidden-when-reading-html) though.)" + "If you get a '403' error, it means that the website has blocked **pandas** because it can see that you are engaged in web scraping. This is because some people web scrape irresponsibly, or because websites have provided other, preferred ways for you to obtain the data, eg via a download of the whole thing (think Wikipedia) or through an API. (If you really need to, [you can often get around the 403 error](https://stackoverflow.com/questions/43590153/http-error-403-forbidden-when-reading-html) though.)\n" ] } ], @@ -543,7 +566,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/whole-game.ipynb b/whole-game.ipynb index eee6c2c..c0b6e5f 100644 --- a/whole-game.ipynb +++ b/whole-game.ipynb @@ -107,7 +107,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/workflow-basics.ipynb b/workflow-basics.ipynb index a0a7e47..64621de 100644 --- a/workflow-basics.ipynb +++ b/workflow-basics.ipynb @@ -528,7 +528,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/workflow-help.quarto_ipynb_1 b/workflow-help.quarto_ipynb_1 new file mode 100644 index 0000000..e7bebf8 --- /dev/null +++ b/workflow-help.quarto_ipynb_1 @@ -0,0 +1,115 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Postscript: Getting Further Help {#sec-workflow-help}\n", + "\n", + "This book is not an island; there is no single resource that will allow you to master Python for Data Science. As you begin to apply the techniques described in this book to your own data, you will soon find questions that we do not answer. This section describes a few tips on how to get help, and to help you keep learning.\n", + "\n", + "## Resources\n", + "\n", + "Some other resources for learning are:\n", + "\n", + "- [The Python Data Science Handbook](https://jakevdp.github.io/PythonDataScienceHandbook/)\n", + "- [Real Python](https://realpython.com/), which has excellent short tutorials that cover Python more broadly (not just data science)\n", + "- [freeCodeCamp's Python courses](https://www.freecodecamp.org/news/search?query=data%20science%20python), though take care to select one that's at the right level for you\n", + "- [Coding for Economists](https://aeturrell.github.io/coding-for-economists), which has similar content to this book but is more in depth and aimed at analysts (particularly in economics)\n", + "\n", + "## Google is your friend\n", + "\n", + "If you get stuck, start with Google. Typically adding \"Python\" or \"Python Data Science\" (as the Python ecosystem goes *well* beyond data science) to a query is enough to restrict it to relevant results. Google is particularly useful for error messages. If you get an error message and you have no idea what it means, try googling it! Chances are that someone else has been confused by it in the past, and there will be help somewhere on the web.\n", + "\n", + "If Google doesn't help, try [Stack Overflow](http://stackoverflow.com). Start by spending a little time searching for an existing answer, including `[Python]` to restrict your search to questions and answers that use Python.\n", + "\n", + "## In the loop\n", + "\n", + "It's also helpful to keep an eye on the latest developments in data science. There are tons of data science newsletters out there, and we recommend keeping up with the Python data science community by following the (#pydata), (#datascience), and (#python) hashtags on Twitter.\n", + "\n", + "## Making a reprex (reproducible example)\n", + "\n", + "If your googling doesn't find anything useful, it's a really good idea prepare a minimal reproducible example or **reprex**.\n", + "\n", + "A good reprex makes it easier for other people to help you, and often you'll figure out the problem yourself in the course of making it. There are two parts to creating a reprex:\n", + "\n", + "- First, you need to make your code reproducible. This means that you need to capture everything, i.e., include any packages you used and create all necessary objects. The easiest way to make sure you've done this is to use the [**watermark**](https://github.com/rasbt/watermark) package alongside whatever else you are doing:" + ], + "id": "22b3f9e0" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import watermark.watermark as watermark\n", + "\n", + "print(watermark())\n", + "print(watermark(iversions=True, globals_=globals()))" + ], + "id": "a119501b", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- Second, you need to make it minimal. Strip away everything that is not directly related to your problem. This usually involves creating a much smaller and simpler Python object than the one you're facing in real life or even using built-in data.\n", + "\n", + "That sounds like a lot of work! And it can be, but it has a great payoff:\n", + "\n", + "- 80% of the time creating an excellent reprex reveals the source of your problem. It's amazing how often the process of writing up a self-contained and minimal example allows you to answer your own question.\n", + "\n", + "- The other 20% of time you will have captured the essence of your problem in a way that is easy for others to play with. This substantially improves your chances of getting help.\n", + "\n", + "There are several things you need to include to make your example reproducible: Python environment, required packages, data, and code.\n", + "\n", + "- **Python environment**--really just the Python version. This is covered by the first call to the **watermark** package.\n", + "\n", + "- **Packages** and their versions. These should be loaded at the top of the script, so it's easy to see which ones the example needs. By using **watermark** with the above configuration, you will also print the package versions. This is a good time to check that you're using the latest version of each package; it's possible you've discovered a bug that's been fixed since you installed or last updated the package.\n", + "\n", + "- **Data**: as others won't be able to easily download the data you're working with, it's often best to create a small amount of data from code that still have the same problem as you're finding with your actual data. Between **numpy** and **pandas**, it's quite easy to generate data from code; here's an example:" + ], + "id": "c4ac60b4" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "df = pd.DataFrame(\n", + " data=np.reshape(range(36), (6, 6)),\n", + " index=[\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"],\n", + " columns=[\"col\" + str(i) for i in range(6)],\n", + " dtype=float,\n", + ")\n", + "df[\"random_normal\"] = np.random.normal(size=6)\n", + "df" + ], + "id": "d1e4562c", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- **Code**: copy and paste the minimal reproducible example code (including the packages, as noted above). Make sure you've used spaces and your variable names are concise, yet informative. Use comments to indicate where your problem lies. Do your best to remove everything that is not related to the problem. Finally, the shorter your code is, the easier it is to understand, and the easier it is to fix.\n", + "\n", + "Finish by checking that you have actually made a reproducible example by starting a fresh Python session and copying and pasting your reprex in." + ], + "id": "4b75e409" + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "language": "python", + "display_name": "Python 3 (ipykernel)", + "path": "/Users/omagic/Documents/GitHub/python4DSpolars/.venv/share/jupyter/kernels/python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/workflow-packages-and-environments.quarto_ipynb_1 b/workflow-packages-and-environments.quarto_ipynb_1 new file mode 100644 index 0000000..a5600ce --- /dev/null +++ b/workflow-packages-and-environments.quarto_ipynb_1 @@ -0,0 +1,149 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Workflow: Packages and Environments {#sec-workflow-packages-and-environments}\n", + "\n", + "In this chapter, you're going to learn about packages and how to install them plus virtual coding environments that keep your packages isolated and your projects reproducible.\n", + "\n", + "## Packages\n", + "\n", + "### Introduction\n", + "\n", + "Packages (also called libraries) are key to extending the functionality of Python. It won't be long before you'll need to install some. There are packages for geoscience, for building websites, for analysing genetic data, for economics—pretty much for anything you can think of. Packages are typically not written by the core maintainers of the Python language but by enthusiasts, firms, researchers, academics, all sorts! Because anyone can write packages, they vary widely in their quality and usefulness. There are some that you'll be seeing them again and again.\n", + "\n", + "

Name a more iconic trio, I'll wait. pic.twitter.com/pGaLuUxQ3r

— Vicki Boykis (\\@vboykis) August 23, 2018
\n", + "\n", + "The three Python packages **numpy**, **pandas**, and **maplotlib**, which respectively provide numerical, data analysis, and plotting functionality, are ubiquitous. So many scripts begin by importing all three of them, as in the tweet above!\n", + "\n", + "There are typically two steps to using a new Python package:\n", + "\n", + "1. *install* the package on the command line (aka the terminal), eg using `uv add pandas`\n", + "\n", + "2. *import* the package into your Python session, eg using `import pandas as pd`\n", + "\n", + "When you issue an install command for a specific package, it is automatically downloaded from the internet and installed in the appropriate place on your computer. To install extra Python packages, you issue install commands to a text-based window called the \"terminal\".\n", + "\n", + "### The Command Line in Brief\n", + "\n", + "The *terminal* or *command line* or sometimes the *command prompt* was labelled 4 in the screenshot of Visual Studio Code from the chapter on @sec-introduction. The terminal is a text-based way to issue all kinds of commands to your computer (not just Python commands) and knowing a little bit about it is really useful for coding (and more) because managing packages, environments (which we haven't yet discussed), and version control (ditto) can all be done via the terminal. We'll come to these in due course in the chapter on @sec-command-line, but for now, a little background on what the terminal is and what it does.\n", + "\n", + "::: {.callout-note}\n", + "To open up the command line within Visual Studio Code, use the ⌃ + \\` keyboard shortcut (Mac) or ctrl + \\` (Windows/Linux), or click \"View > Terminal\".\n", + "\n", + "If you want to open up the command line independently of Visual Studio Code, search for \"Terminal\" on Mac and Linux, and \"Powershell\" on Windows.\n", + ":::\n", + "\n", + "Firstly, everything you can do by clicking on icons to launch programmes on your computer, you can also do via the terminal, also known as the command line. For many programmes, a lot of their functionality can be accessed using the command line, and other programmes *only* have a command line interface (CLI), including some that are used for data science.\n", + "\n", + "::: {.callout-tip}\n", + "The command line interacts with your operating system and is used to create, activate, or change Python installations.\n", + ":::\n", + "\n", + "Use Visual Studio Code to open a terminal window by clicking Terminal -> New Terminal on the list of commands at the very top of the window. If you have installed uv on your computer, your terminal should look something like this as your 'command prompt':\n", + "\n", + "```bash\n", + "your-username@your-computer current-directory %\n", + "```\n", + "\n", + "on Mac, and the same but with '%' replaced by '$' on linux, and (using Powershell)\n", + "\n", + "```powershell\n", + "PS C:\\Windows\\System32>\n", + "```\n", + "\n", + "on Windows.\n", + "\n", + "You can check that uv has successfully installed Python in your current project's folder by running\n", + "\n", + "```bash\n", + "uv run python --version\n", + "```\n", + "\n", + "For now, to at least try out the command line, let's use something that works across all three of the major operating systems. Type `uv run python` on the command prompt that came up in your new terminal window. You should see information about your installation of Python appear, including the version, followed by a Python prompt that looks like `>>>`. This is a kind of interactive Python session, in the terminal. It's much less rich than the one available in Visual Studio Code (it can't run scripts line-by-line, for example) but you can try `print('Hello World!')` and it will run, printing your message. To exit the terminal-based Python session, type `exit()` to go back to the regular command line.\n", + "\n", + "### Installing Packages\n", + "\n", + "To install extra Python packages, the default and easiest way is to use `uv add **packagename**`. There are over 330,000 Python packages on PyPI (the Python Package Index)! You can see what packages you have installed already by running `uv pip list` into the command line.\n", + "\n", + "`uv add ...` will install packages into the special Python environment in your current folder (it sits in a subdirectory called \".venv\" which will be hidden by default on most systems.) It's really helpful and good practice to have one Python environment per project, and **uv** does this automatically for you.\n", + "\n", + "::: {.callout-tip title=\"Exercise\"}\n", + "Try installing the **matplotlib**, **pandas**, **statsmodels**, and **skimpy** packages using `uv add`.\n", + ":::\n", + "\n", + "### Using Packages\n", + "\n", + "Once you have installed a package, you need to be able to use it! This is usually done via an import statement at the top of your script or Jupyter Notebook. For example, to bring in **pandas**, it's\n", + "\n", + "```python\n", + "import pandas as pd\n", + "```\n", + "\n", + "Why does Python do this? The idea of not just loading every package is to provide clarity over what function is being called from what package. It's also not necessary to load every package for every piece of analysis, and you often actually want to know what the *minimum* set of packages is to reproduce an analysis. Making the package imports explicit helps with all of that.\n", + "\n", + "You may also wonder why one doesn't just use `import pandas as pandas`. There's actually nothing stopping you doing this except i) it's convenient to have a shorter name and ii) there does tend to be a convention around imports, ie `pd` for **pandas** and `np` for **numpy**, and your code will be clearer to yourself and others if you follow the conventions.\n", + "\n", + "## Virtual Code Environments\n", + "\n", + "Virtual code environments allow you to isolate all of the packages that you're using to do analysis for one project from the set of packages you might need for a different project. They're an important part of creating a reproducible analytical pipeline but a key benefit is that others can reproduce the environment you used and it's best practice to have an isolated environment per project.\n", + "\n", + "To be more concrete, let's say you're using Python 3.9, **statsmodels**, and **pandas** for one project, project A. And, for project B, you need to use Python 3.10 with **numpy** and **scikit-learn**. Even with the same version of Python, best practice would be to have two separate virtual Python environments: environment A, with everything needed for project A, and environment B, with everything needed for project B. For the case where you're using different versions of Python, this isn't just best practice, it's essential.\n", + "\n", + "Many programming languages now come with an option to install packages and a version of the language in isolated environments. In Python, there are multiple tools for managing different environments. And, of those, the easiest to work with is probably [**uv**](https://docs.astral.sh/uv/).\n", + "\n", + "You can see all of the packages in the environment created in your current folder by running `uv pip list` on the command line. Here's an example of looking at the installed packages within this very book, filtering them just to the ones beginning with \"s\".\n", + "\n", + "```{bash}\n", + "uv run pip list | grep ^s\n", + "```\n", + "\n", + "### The pyproject.toml file in Python Environments\n", + "\n", + "You may have noticed that a file called `pyproject.toml` has been created." + ], + "id": "8b889898" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import toml\n", + "from rich import print_json\n", + "\n", + "print_json(data=toml.load(\"pyproject.toml\"))" + ], + "id": "688f09f1", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This lists all of the dependencies, and the version, of a **uv** Python project. There are lots of benefits to tracking what versions of packages you're using like this. One of the most important is that you can *share* projects with other people, and they can install them from these files too.\n", + "\n", + "As you install or remove packages, the `pyproject.toml` file changes in lockstep.\n", + "\n", + "Noe that Visual Studio Code shows which Python environment you are using when you open a Python script or Jupyter Notebook.\n", + "\n", + "![A typical user view in Visual Studio Code](https://github.com/aeturrell/coding-for-economists/blob/main/img/vscode_layout.png?raw=true)\n", + "\n", + "In the screenshot above, you can see the project-environment in two places: on the blue bar at the bottom of the screen, and (in 5), at the top right hand side of the interactive window. A similar top right indicator is present when you have a Jupyter Notebook open too." + ], + "id": "148595b3" + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "language": "python", + "display_name": "Python 3 (ipykernel)", + "path": "/Users/omagic/Documents/GitHub/python4DSpolars/.venv/share/jupyter/kernels/python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/workflow-style.ipynb b/workflow-style.ipynb index ab2eb53..3de114b 100644 --- a/workflow-style.ipynb +++ b/workflow-style.ipynb @@ -9,11 +9,11 @@ "\n", "Good coding style is like correct punctuation: you can manage without it, butitsuremakesthingseasiertoread. Even as a very new programmer it's a good idea to work on your code style. Use a consistent style makes it easier for others (including future-you!) to read your work, and is particularly important if you need to get help from someone else.\n", "\n", - "This chapter will introduce you to some important style points drawn from [Clean Code in Python](https://testdriven.io/blog/clean-code-python/), [Tips for Better Coding](https://aeturrell.github.io/coding-for-economists/code-best-practice.html) from *Coding for Economists*, the UK Government Statistical Service’s [Quality Assurance of Code for Analysis and Research](https://best-practice-and-impact.github.io/qa-of-code-guidance/intro.html) guidance, and the bible of Python style guides, [PEP 8 — Style Guide for Python Code](https://peps.python.org/pep-0008/).\n", + "This chapter will introduce you to some important style points drawn from [Clean Code in Python](https://testdriven.io/blog/clean-code-python/), [Tips for Better Coding](https://aeturrell.github.io/coding-for-economists/code-best-practice.html) from _Coding for Economists_, the UK Government Statistical Service’s [Quality Assurance of Code for Analysis and Research](https://best-practice-and-impact.github.io/qa-of-code-guidance/intro.html) guidance, and the bible of Python style guides, [PEP 8 — Style Guide for Python Code](https://peps.python.org/pep-0008/).\n", "\n", "Styling your code will feel a bit tedious to start with, but if you practice it, it will soon become second nature. Additionally, there are some great tools to quickly restyle existing code, like the [Black](https://black.readthedocs.io/) Python package (\"you can have any colour you like, as long as it's black\").\n", "\n", - "Once you've installed Black by running `uv tool install black`, you can use it on the command line (aka the terminal) within Visual Studio Code. Open up a terminal by clicking 'Terminal -> New Terminal' and then run `black *.py` to apply a standard code style to all Python scripts in the current directory." + "Once you've installed Black by running `uv tool install black`, you can use it on the command line (aka the terminal) within Visual Studio Code. Open up a terminal by clicking 'Terminal -> New Terminal' and then run `black *.py` to apply a standard code style to all Python scripts in the current directory.\n" ] }, { @@ -23,44 +23,47 @@ "source": [ "## Names\n", "\n", - "First, names matter. Use meaningful names for variables, functions, or whatever it is you're naming. Avoid abbreviations that you understand *now* but which will be unclear to others, or future you. For example, use `real_wage_hourly` over `re_wg_ph`. I know it's tempting to use `temp` but you'll feel silly later when you can't for the life of you remember what `temp` does or is. A good trick when naming booleans (variables that are either true or false) is to use `is` followed by what the boolean variable refers to, for example `is_married`.\n", + "First, names matter. Use meaningful names for variables, functions, or whatever it is you're naming. Avoid abbreviations that you understand _now_ but which will be unclear to others, or future you. For example, use `real_wage_hourly` over `re_wg_ph`. I know it's tempting to use `temp` but you'll feel silly later when you can't for the life of you remember what `temp` does or is. A good trick when naming booleans (variables that are either true or false) is to use `is` followed by what the boolean variable refers to, for example `is_married`.\n", "\n", "As well as this general tip, Python has conventions on naming different kinds of variables. The naming convention for almost all objects is lower case separated by underscores, e.g. `a_variable=10` or ‘this_is_a_script.py’. This style of naming is also known as snake case. There are different naming conventions though—[Allison Horst](https://twitter.com/allison_horst) made this fantastic cartoon of the different conventions that are in use.\n", "\n", - "![Different naming conventions. Artwork by \\@allison_horst.](https://github.com/aeturrell/coding-for-economists/raw/main/img/in_that_case.jpg) Different naming conventions. Artwork by \\@allison_horst.\n", + "![Different naming conventions. Artwork by @allison_horst.](https://github.com/aeturrell/coding-for-economists/raw/main/img/in_that_case.jpg) Different naming conventions. Artwork by \\@allison_horst.\n", "\n", "There are three exceptions to the snake case convention: classes, which should be in camel case, eg `ThisIsAClass`; constants, which are in capital snake case, eg `THIS_IS_A_CONSTANT`; and packages, which are typically without spaces or underscores and are lowercase `thisisapackage`.\n", "\n", - "For some quick shortcuts to re-naming columns in **pandas** data frames or other string variables, try the unicode-friendly [**slugify**](https://github.com/un33k/python-slugify) library or the `clean_headers()` function from the [**dataprep**](https://docs.dataprep.ai/index.html) library.\n", + "For some quick shortcuts to re-naming columns in **polars** data frames or other string variables, try the unicode-friendly [**slugify**](https://github.com/un33k/python-slugify) library or the `clean_headers()` function from the [**dataprep**](https://docs.dataprep.ai/index.html) library.\n", "\n", "The better named your variables, the clearer your code will be--and the fewer comments you will need to write!\n", "\n", "In summary:\n", + "\n", "- use descriptive variable names that reveal your intention, eg `days_since_treatment`\n", "- avoid using ambiguous abbreviations in names, eg use `real_wage_hourly` over `rw_ph`\n", "- always use the same vocabulary, eg don't switch from `worker_type` to `employee_type`\n", "- avoid 'magic numbers', eg numbers in your code that set a key parameter. Set these as named constants instead. Here's an example:\n", - " ```python\n", - " import random\n", "\n", - " # This is bad\n", - " def roll():\n", - " return random.randint(0, 36) # magic number!\n", + " ```python\n", + " import random\n", + "\n", + " # This is bad\n", + " def roll():\n", + " return random.randint(0, 36) # magic number!\n", "\n", - " # This is good\n", - " MAX_INT_VALUE = 36\n", + " # This is good\n", + " MAX_INT_VALUE = 36\n", + "\n", + " def roll():\n", + " return random.randint(0, MAX_INT_VALUE)\n", + " ```\n", "\n", - " def roll():\n", - " return random.randint(0, MAX_INT_VALUE)\n", - " ```\n", "- use verbs for function names, eg `get_regression()`\n", "- use consistent verbs for function names, don't use `get_score()` and `grab_results()` (instead use `get` for both)\n", "- variable names should be snake_case and all lowercase, eg `first_name`\n", - " - class names should be CamelCase, eg `MyClass`\n", - " - function names should be snake_case and all lowercase, eg `quick_sort()`\n", - " - constants should be snake_case and all uppercase, eg `PI = 3.14159`\n", - " - modules should have short, snake_case names and all lowercase, eg `pandas`\n", - " - single quotes and double quotes are equivalent so pick one and be consistent—most automatic formatters prefer `\"`" + "- class names should be CamelCase, eg `MyClass`\n", + "- function names should be snake_case and all lowercase, eg `quick_sort()`\n", + "- constants should be snake_case and all uppercase, eg `PI = 3.14159`\n", + "- modules should have short, snake_case names and all lowercase, eg `pandas`\n", + "- single quotes and double quotes are equivalent so pick one and be consistent—most automatic formatters prefer `\"`\n" ] }, { @@ -88,7 +91,7 @@ "list_var = [1,2,3,4]\n", "# or\n", "list_var = [1 , 2 , 3 , 4]\n", - "```" + "```\n" ] }, { @@ -98,9 +101,9 @@ "source": [ "## Code Comments\n", "\n", - "As mentioned previously, Python will ignore any text after `#`. This allows to you to write **comments**, text that is ignored by Python but can be read by other humans. Comments can be helpful for briefly describing what the subsequent code does: use them to provide extra contextual information that *isn't* conveyed by function and variable names.\n", + "As mentioned previously, Python will ignore any text after `#`. This allows to you to write **comments**, text that is ignored by Python but can be read by other humans. Comments can be helpful for briefly describing what the subsequent code does: use them to provide extra contextual information that _isn't_ conveyed by function and variable names.\n", "\n", - "Actually, well-written code needs *fewer* comments because it's more evidence what's going on. And it's tempting not to update comments even when code changes. So do comment, but see if you can make the code tell its own story first.\n", + "Actually, well-written code needs _fewer_ comments because it's more evidence what's going on. And it's tempting not to update comments even when code changes. So do comment, but see if you can make the code tell its own story first.\n", "\n", "Also, avoid \"noise\" comments that tell you what you already know from just looking at the code.\n", "\n", @@ -109,17 +112,16 @@ "Finally, functions come with their own special type of comments called a doc string. Here's an example that tells us all about the functions inputs and outputs, including the type of input and output (here a data frame, also known as `pd.DataFrame`).\n", "\n", "```python\n", - "def round_dataframe(df: pd.DataFrame) -> pd.DataFrame:\n", - " \"\"\"Rounds numeric columns in dataframe to 2 s.f.\n", + "def get_numeric_mean(df: pl.DataFrame) -> pl.DataFrame:\n", + " \"\"\"Returns the mean of each numeric column in the DataFrame.\n", " Args:\n", - " df (pd.DataFrame): Input dataframe\n", + " df (pl.DataFrame): Input dataframe\n", " Returns:\n", - " pd.DataFrame: Dataframe with numbers rounded to 2 s.f.\n", + " pl.DataFrame: Dataframe of mean value of each numeric column.\n", " \"\"\"\n", - " for col in df.select_dtypes(\"number\"):\n", - " df[col] = df[col].apply(lambda x: float(f'{float(f\"{x:.2g}\"):g}'))\n", - " return df\n", - "```" + " df_mean = df.select(S.numeric()).mean()\n", + " return df_mean\n", + "```\n" ] }, { @@ -141,7 +143,7 @@ " return result\n", "```\n", "\n", - "When using *method chaining* (something you can see in action in [](data-transform)) it's necessary to put the chain inside parentheses and it's good practice to use a new line for every method. The code snippet below gives an example of what good looks like:" + "When using _method chaining_ (something you can see in action in [](data-transform)) it's necessary to put the chain inside parentheses and it's good practice to use a new line for every method. The code snippet below gives an example of what good looks like:\n" ] }, { @@ -151,9 +153,9 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "import polars as pl\n", "\n", - "df = pd.DataFrame(\n", + "df = pl.DataFrame(\n", " data={\n", " \"col0\": [0, 0, 0, 0],\n", " \"col1\": [0, 0, 0, 0],\n", @@ -161,13 +163,14 @@ " \"col3\": [\"a\", \"b\", \"b\", \"a\"],\n", " \"col4\": [\"alpha\", \"gamma\", \"gamma\", \"gamma\"],\n", " },\n", - " index=[\"row\" + str(i) for i in range(4)],\n", ")\n", "\n", "\n", "# Chaining inside parentheses works\n", "\n", - "results = df.groupby([\"col3\", \"col4\"]).agg({\"col1\": \"count\", \"col2\": \"mean\"})\n", + "results = df.group_by([\"col3\", \"col4\"]).agg(\n", + " pl.col(\"col1\").count(), pl.col(\"col2\").mean()\n", + ")\n", "\n", "results" ] @@ -177,13 +180,13 @@ "id": "1d6f3bf8", "metadata": {}, "source": [ - "And this is what *not* to do:\n", + "And this is what _not_ to do:\n", "\n", "```python\n", "results = df\n", - " .groupby([\"col3\", \"col4\"]).agg({\"col1\": \"count\", \"col2\": \"mean\"})\n", + " .group_by([\"col3\", \"col4\"]).agg(pl.col(\"col1\").count(), pl.col(\"col2\").mean())\n", "\n", - "```" + "```\n" ] }, { @@ -193,7 +196,7 @@ "source": [ "## Principles of Clean Code\n", "\n", - "While automation can help apply style, it can't help you write *clean code*. Clean code is a set of rules and principles that helps to keep your code readable, maintainable, and extendable. Writing code is easy; writing clean code is hard! However, if you follow these principles, you won't go far wong.\n", + "While automation can help apply style, it can't help you write _clean code_. Clean code is a set of rules and principles that helps to keep your code readable, maintainable, and extendable. Writing code is easy; writing clean code is hard! However, if you follow these principles, you won't go far wong.\n", "\n", "### Do not repeat yourself (DRY)\n", "\n", @@ -239,7 +242,7 @@ "\n", "def lowercase(text):\n", " return text.lower()\n", - "```" + "```\n" ] } ], @@ -268,7 +271,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" }, "toc-showtags": true }, diff --git a/workflow-writing-code.ipynb b/workflow-writing-code.ipynb index 28e2edd..bb442d2 100644 --- a/workflow-writing-code.ipynb +++ b/workflow-writing-code.ipynb @@ -150,7 +150,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.0" + "version": "3.12.13" } }, "nbformat": 4,