{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Liczenie słów w „Panu Tadeuszu” w Databricks\n",
    "Plik `pan_tadeusz.txt` wgrywamy do wolumenu Unity Catalog `workspace.historia_big_data.pliki`\n",
    "(Catalog, potem workspace, historia_big_data, pliki, Upload to this volume)."
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "spark.sql(\"CREATE SCHEMA IF NOT EXISTS workspace.historia_big_data\")\n",
    "spark.sql(\"CREATE VOLUME IF NOT EXISTS workspace.historia_big_data.pliki\")"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "from pyspark.sql import functions as F\n",
    "\n",
    "lines = spark.read.text(\"/Volumes/workspace/historia_big_data/pliki/pan_tadeusz.txt\")\n",
    "words = (lines\n",
    "         .select(F.explode(F.split(F.lower(\"value\"), r\"[^\\p{L}]+\")).alias(\"word\"))\n",
    "         .where(F.col(\"word\") != \"\"))\n",
    "counts = words.groupBy(\"word\").count().orderBy(F.desc(\"count\"), \"word\")\n",
    "\n",
    "counts.write.mode(\"overwrite\").saveAsTable(\"workspace.historia_big_data.word_counts\")\n",
    "display(counts.limit(20))"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "%sql\n",
    "DESCRIBE HISTORY workspace.historia_big_data.word_counts"
   ],
   "execution_count": null,
   "outputs": []
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  },
  "kernelspec": {
   "name": "synapse_pyspark",
   "display_name": "PySpark",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}