Move to Selenium

.github/workflows/download_registers_of_interests.yaml

@@ -15,6 +15,19 @@ jobs:
       - uses: r-lib/actions/setup-r@v2
       - uses: r-lib/actions/setup-pandoc@v2
 
+      - name: Install Java 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: '17'
+
+      - uses: nanasess/setup-chromedriver@v2
+
+      - name: Download and start Selenium
+        run: |
+          wget https://selenium-release.storage.googleapis.com/3.9/selenium-server-standalone-3.9.1.jar
+          java -jar selenium-server-standalone-3.9.1.jar &
+
       - name: Query dependencies
         run: |
           install.packages('remotes')

R/get_html_for_url.R

@@ -0,0 +1,12 @@
+get_html_for_url <- function(url, sleep=0){
+
+  remDr <- get_selenium_session()
+
+  remDr$navigate(url)
+
+  if(sleep > 0){
+    Sys.sleep(sleep)
+  }
+
+  remDr$getPageSource() |> dplyr::first() |> xml2::read_html()
+}

R/get_info_from_ms_page_url.R

@@ -1,5 +1,5 @@
 get_info_from_ms_page_url <- function(ms_page_url, ms_welsh_page_url=NULL){
-  ms_page <- read_html(ms_page_url)
+  ms_page <- get_html_for_url(ms_page_url)
 
   ret_info <- list()
 

R/get_interests_from_roi_page_url.R

@@ -1,6 +1,6 @@
 get_interests_from_roi_page_url = function(roi_page_url){
 
-  roi_page <- read_html(roi_page_url)
+  roi_page <- get_html_for_url(roi_page_url)
   roi_tables <- roi_page %>% html_nodes(".mgInterestsTable")
 
   ret <- tibble(

R/get_link_to_register_from_ms_page_url.R

@@ -1,5 +1,5 @@
 # Manual fallback - generally you can just use https://business.senedd.wales/mgRofI.aspx?UID=
 get_link_to_register_from_ms_page_url <- function(ms_page_url){
   ms_page <- read_html(ms_page_url)
-  ms_page %>% html_node(xpath='//a[contains(text(), "View Register")]') %>% html_attr("href")
+  ms_page %>% get_html_for_url(xpath='//a[contains(text(), "View Register")]') %>% html_attr("href")
 }

R/get_selenium_session.R

@@ -0,0 +1,31 @@
+get_selenium_session_impl <- function() {
+  sess <- NULL
+
+  function (force_new = FALSE){
+    if(!is.null(sess) && !force_new){
+      return(sess)
+    }
+
+    remDr <- RSelenium::remoteDriver(
+      remoteServerAddr = "localhost",
+      port = 4444L,
+      browserName = "chrome",
+      extraCapabilities = list(
+        chromeOptions = list(
+          args = list(
+            "--headless=new",
+            "--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
+            "--disable-gpu",
+            "--disable-dev-shm-usage",
+            "--disable-extensions")))
+    )
+
+    remDr$open()
+
+    sess <<- remDr
+
+    return(sess)
+  }
+}
+
+get_selenium_session <- get_selenium_session_impl()

R/get_senedd_members.R

@@ -13,7 +13,7 @@ get_senedd_members <- function(extra_info=FALSE){
     }
 
     try(
-      member_list_html <- read_html("https://senedd.wales/find-a-member-of-the-senedd/?VW=Table&PageSize=10000&Page=1&Culture=en-GB&IsSubSearch=False&IsPostcodeCrossConstituency=False&Postcode=&Name=&ShowAll=true&Region=&Constituency=&Constituency=&Constituency=&Constituency=&Constituency=&PartyFilterType=party&PoliticalParty=&PoliticalPartyGroup=&partyValueName=") 
+      member_list_html <- get_html_for_url("https://senedd.wales/find-a-member-of-the-senedd/?isActiveMs=true", 0.5) 
     )
   }
 
@@ -43,6 +43,7 @@ get_senedd_members <- function(extra_info=FALSE){
     ))
   }
 
+
   ret <- ret %>% mutate(
     Constituency = if_else(ConstituencyOrRegion %in% senedd_constituencies()$SeneddConstituencyName, ConstituencyOrRegion, NA_character_),
     Region = if_else(ConstituencyOrRegion %in% senedd_constituencies()$SeneddRegionName, ConstituencyOrRegion, NA_character_)