Topics in the Bundestag (Part 2)

# install a current version of Plotly and kaleido
!pip install plotly==5.3.0 kaleido

# import required Python packages
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
import plotly.io as pio
pio.templates.default = "plotly_white"
from plotly.subplots import make_subplots
import textwrap

# mount Google Drive (only needed if you're using Google Colab)
from google.colab import drive
drive.mount("/content/gdrive")

# load dataset into memory
classified_speech_paragraphs_df = pd.read_parquet("/content/gdrive/My Drive/Colab Notebooks/bundestag_plenary_protocols_with_multiclass_topics_term_19.parquet")

# add statistics on speech length
classified_speech_paragraphs_df["speech_len_paragraphs"] = classified_speech_paragraphs_df.groupby("speech_id")["speech_id"].transform("count")
classified_speech_paragraphs_df["speech_len_words"] = classified_speech_paragraphs_df.groupby("speech_id")["paragraph_len_words"].transform("sum")
classified_speech_paragraphs_df["speech_len_chars"] = classified_speech_paragraphs_df.groupby("speech_id")["paragraph_len_chars"].transform("sum")

classified_speech_paragraphs_df.sample(5)

print(f"Number of paragraphs:\t\t{len(classified_speech_paragraphs_df)}")
print(f"Median words per paragraph:\t{classified_speech_paragraphs_df.paragraph_len_words.median()}")
print(f"Mean words per paragraph:\t{classified_speech_paragraphs_df.paragraph_len_words.mean()}")
print(f"Number of speeches:\t\t{len(classified_speech_paragraphs_df.speech_id.unique())}")
print(f"Number of sessions (dates):\t{len(classified_speech_paragraphs_df.date.unique())}")

# group by fraction and calculate mean topic probabilites for each group
fractions_topics_df = classified_speech_paragraphs_df.groupby("speaker_fraction").mean().iloc[:, 3:16]

# function for creating overlayed fraction topic profile plots
def fractions_topics_plot_overlayed(fig_df, fractions_colors={"CDU/CSU": "black", "SPD": "red", "FDP": "gold"}, title="untitled"):
  fig_long_df = fig_df.reset_index().melt(id_vars=["speaker_fraction"]) # transform tidy (wide) df to long form df
  fig = go.Figure()
  for (fraction, color) in fractions_colors.items():
    fig.add_trace(go.Scatterpolar(theta=fig_long_df.query(f"speaker_fraction=='{fraction}'")["variable"],
                                  r=fig_long_df.query(f"speaker_fraction=='{fraction}'")["value"],
                                  mode="lines+markers",
                                  line=dict(shape="spline"),
                                  fill="toself",
                                  name=f"{fraction}",
                                  line_color=f"{color}"))
  fig.update_layout(title=title, showlegend=True, width=800, height=600)
  return fig

# create overlayed fraction topic profiles for the three largest fractions
fractions_topics_plot_overlayed(fractions_topics_df,
                                fractions_colors={"CDU/CSU": "black", "SPD": "red", "FDP": "gold"},
                                title="Topics in the 19th Bundestag, by Fraction")

fractions_topics_nrm_df = (fractions_topics_df - fractions_topics_df.mean()) / fractions_topics_df.std()

# create overlayed normalized fraction topic profiles for the three largest fractions
fractions_topics_plot_overlayed(fractions_topics_nrm_df,
                                fractions_colors={"CDU/CSU": "black", "SPD": "red", "FDP": "gold"},
                                title="Topics in the 19th Bundestag, Normalized, by Fraction")

# function to create a trellis plot of all fraction topic profiles
def fractions_topics_plot_trellis(fig_df, title="untitled"):
  fig_long_df = fig_df.reset_index().melt(id_vars=["speaker_fraction"]) # transform tidy (wide) df to long form df
  
  def fraction_profile_plot(fraction, color):
    return go.Scatterpolar(theta=fig_long_df.query(f"speaker_fraction=='{fraction}'")["variable"],
                           r=fig_long_df.query(f"speaker_fraction=='{fraction}'")["value"],
                           mode="lines+markers",
                           line=dict(shape="spline"),
                           fill="toself",
                           name=f"{fraction}",
                           line_color=color)

  fig = make_subplots(rows=3, cols=2,
                      #subplot_titles=("CDU/CSU", "SPD", "AfD", "FDP", "DIE LINKE", "BÜNDNIS 90/DIE GRÜNEN"),
                      specs=[[{"type": "polar"}, {"type": "polar"}],
                             [{"type": "polar"}, {"type": "polar"}],
                             [{"type": "polar"}, {"type": "polar"}]])
  # CDU/CSU
  fig.add_trace(fraction_profile_plot(fraction="CDU/CSU", color="black"), row=1, col=1)
  # SPD
  fig.add_trace(fraction_profile_plot(fraction="SPD", color="red"), row=1, col=2)
  # AfD
  fig.add_trace(fraction_profile_plot(fraction="AfD", color="blue"), row=2, col=1)
  # FDP
  fig.add_trace(fraction_profile_plot(fraction="FDP", color="gold"), row=2, col=2)
  # DIE LINKE
  fig.add_trace(fraction_profile_plot(fraction="DIE LINKE", color="purple"), row=3, col=1)
  # BÜNDNIS 90/DIE GRÜNEN
  fig.add_trace(fraction_profile_plot(fraction="BÜNDNIS 90/DIE GRÜNEN", color="green"), row=3, col=2)
  fig.update_polars(radialaxis=dict(range=[fig_long_df.min()[2], fig_long_df.max()[2]]))
  fig.update_layout(title=title,
                    showlegend=True,
                    autosize=False,
                    width=1200,
                    height=1600)
  return fig

fractions_topics_plot_trellis(fractions_topics_nrm_df, title="Topics in the 19th Bundestag, Normalized, by Fraction")

# keep date and probability columns
date_scores_df = classified_speech_paragraphs_df.iloc[:, [1] + list(range(13, 26))]
# group by date and calculate mean probabilites
date_mean_scores_df = date_scores_df.groupby("date").mean().reset_index()

# plot date against mean probabilities
px.line(date_mean_scores_df.melt(id_vars=["date"]).reset_index(),
        x="date", y="value",
        color="variable", color_discrete_sequence=px.colors.qualitative.Alphabet_r,
        markers=True,
        title="Topics in the 19th Bundestag, by Session Date")

# apply a 30-day moving average
date_mean_scores_ma30_df = date_mean_scores_df.set_index("date").rolling(30).mean().reset_index()

# plot, as above
px.line(date_mean_scores_ma30_df.melt(id_vars=["date"]).reset_index(),
              x="date", y="value",
              color="variable", color_discrete_sequence=px.colors.qualitative.Alphabet_r,
              markers=True,
              title="Topics in the 19th Bundestag, by Session Date")

# it would be cleaner to group by speaker_id, but this is simpler (as long as no speakers share the same full name and fraction)
speaker_scores_df = classified_speech_paragraphs_df.iloc[:, [7] + list(range(13, 26))]
speaker_mean_scores_df = speaker_scores_df.groupby("speaker_full_name").mean().reset_index()

def top10(df, id_column, topic):
  return df.nlargest(10, topic).loc[:, [id_column, topic]].reset_index()

top10(speaker_mean_scores_df, "speaker_full_name", "Außenpolitik")

top10(speaker_mean_scores_df, "speaker_full_name", "Klima")

fraction_scores_df = classified_speech_paragraphs_df.iloc[:, [6] + list(range(13, 26))]
fraction_mean_scores_df = fraction_scores_df.groupby("speaker_fraction").mean().reset_index()

top10(fraction_mean_scores_df, "speaker_fraction", "Außenpolitik")

top10(fraction_mean_scores_df, "speaker_fraction", "Klima")

def speeches_for_topic_combination(classified_speech_paragraphs_df,
                                   min_speech_len_words=140,
                                   topic_combination=["Klima", "Jobs"]):
  # create dataframe of speeches with scores, augmented with speaker, speech_len_words, and sample_paragraph...
  speech_speaker_scores_df = (classified_speech_paragraphs_df.iloc[:, [8] + list(range(13, 26))]
                              .groupby(["speech_id"])
                              .mean()
                              .join(classified_speech_paragraphs_df # augment with full speech texts...
                                    .loc[:, ["speech_id", "paragraph", "paragraph_num"]]
                                    .sort_values(["speech_id", "paragraph_num"])
                                    .groupby("speech_id")
                                    .agg({"paragraph": "\n\n".join}), how="left") # TODO how to separate paragraphs in string?
                              .join(classified_speech_paragraphs_df # augment with speaker name and speech length...
                                    .loc[:, ["speech_id", "speaker_full_name", "speech_len_words"]]
                                    .drop_duplicates(subset=["speech_id"])
                                    .set_index("speech_id"), how="left")
                              .reset_index()
                              .rename(columns={"paragraph": "speech"})
                              .query(f"speech_len_words>={min_speech_len_words}"))
  # augment with topic combination score (product of individual topic scores)...
  topic_combination_str = "_".join(topic_combination)
  topic_combination_product_expr = " * ".join([f"speech_speaker_scores_df.{t}" for t in topic_combination])
  speech_speaker_scores_df[topic_combination_str] = pd.eval(topic_combination_product_expr)
  return speech_speaker_scores_df

def topN_speeches_for_topic_combination(classified_speech_paragraphs_df,
                                        N=10,
                                        min_speech_len_words=140,
                                        topic_combination=["Klima", "Jobs"]):
  sftc = speeches_for_topic_combination(classified_speech_paragraphs_df,
                                        min_speech_len_words=min_speech_len_words,
                                        topic_combination=topic_combination)
  topic_combination_str = "_".join(topic_combination)
  return (sftc
          .nlargest(N, topic_combination_str)
          .loc[:, ["speech_id", "speaker_full_name", "speech_len_words", "speech", topic_combination_str]])

pd.set_option("display.max_colwidth", None)
topN_speeches_for_topic_combination(classified_speech_paragraphs_df, topic_combination=["Klima", "Jobs"])

pd.set_option("display.max_colwidth", None)
topN_speeches_for_topic_combination(classified_speech_paragraphs_df, topic_combination=["Bildung", "Zuwanderung", "Jobs"])

def speech_topic_time_series_plot(classified_speech_paragraphs_df, speech_id):
  classified_speech_paragraphs_df["paragraph_end_word_pos"] = (classified_speech_paragraphs_df
                                                               .paragraph_len_words
                                                               .cumsum())
  fig_df = classified_speech_paragraphs_df.query(f"speech_id=='{speech_id}'").copy()
  fig_df["paragraph_end_word_pos"] = fig_df.paragraph_len_words.cumsum()
  fig_df = (fig_df
            .iloc[:, [10, 29] + list(range(13, 26))]
            .melt(id_vars=["paragraph_end_word_pos", "paragraph"])
            .reset_index()
            .rename(columns={"variable": "topic"}))
  fig_df["paragraph"] = fig_df.paragraph.apply(lambda p: "<br>".join(textwrap.wrap(p, width=70)))
  return px.line(fig_df,
                 x="paragraph_end_word_pos", y="value",
                 color="topic", color_discrete_sequence=px.colors.qualitative.Alphabet_r,
                 hover_data=["paragraph", "topic", "value"],
                 markers=True,
                 title=f"Topics of Speech {speech_id}, by Paragraph")

speech_topic_time_series_plot(classified_speech_paragraphs_df, "ID1910504900")