FrmMainApp.cs

using CsvHelper;
using HLWebScraper.Net.Helpers;
using HLWebScraper.Net.Model;
using HtmlAgilityPack;
using Newtonsoft.Json;
using System.Diagnostics;
using System.Globalization;
using WinFormsDarkThemerNinja;
using HtmlDocument = HtmlAgilityPack.HtmlDocument;

#pragma warning disable CA1416

namespace HLWebScraper.Net;

public partial class FrmMainApp : Form
{
    private const string FxUrl = "https://www.floatrates.com/daily/gbp.json";
    private const string Not_ETF_ETFType = "## Not ETF";
    internal static readonly FXCurrencyCollection FxCurrencies = [];
    private static char[] selectedAlphabet;

    private static readonly HashSet<string> UrlListOfStocksAndShares = [];
    private static readonly CompressedMemoryCache<Dictionary<string, string>> urlAndHtmlContentHashtable = new();
    private static readonly CompressedMemoryCache<Dictionary<string, string>> urlAndCompanyInfoHashtable = new();

    private static int _urlCounter;
    private static readonly object urlCounterLock = new();

    private static readonly HashSet<SEDOL> SEDOLs = [];

    internal static IEnumerable<ETFType>? ETF_Types;

    private CancellationTokenSource cancellationTokenSource;
    private static int _maxConnectionsPerServerSetting;
    private static readonly HttpClient HttpClient = new();

    public FrmMainApp()
    {
        cancellationTokenSource = new CancellationTokenSource();

        InitializeComponent();
        GetETFTypesFromCSV(); // read user-defined ETF types
        HelperDataDatabaseAndStartup.DataCreateSQLiteDB(); // make sure sqlite database exists (create if necessary)
        HelperDataDatabaseAndStartup.DataWriteSQLiteSettingsDefaultSettings(); // fill db w/ defaults where needed

        // get defaults 

        // re _maxConnectionsPerServerSetting: this is counted 2x in the code so a setting of 50 will send out 100 requests because
        // ... we're basically sending 1 for the "normal" page and 1 for the "coporate info" page.
        _maxConnectionsPerServerSetting = Convert.ToInt16(
            value: HelperDataApplicationSettings.DataReadSQLiteSettings(tableName: "settings",
                settingId: "MaxConnectionsPerServer"));
    }

    private void FrmMainApp_Load(object sender, EventArgs e)
    {
        if (HelperDataApplicationSettings.DataReadSQLiteSettings(tableName: "settings",
                settingId: "Theme") == "Dark")
        {
            tsmi_DarkishMode.PerformClick();
        }

        btn_StartScrape.Enabled = true;
        btn_Stop.Enabled = false;
        btn_ReloadCategories.Enabled = false;
        btn_SaveToCSV.Enabled = false;
        tpg_Overview.Enabled = false;

#if DEBUG
        lbx_Alphabet.SetSelected(index: lbx_Alphabet.FindStringExact(s: "y"), value: true);
#else
        for (int i = 0; i < lbx_Alphabet.Items.Count; i++)
        {
            lbx_Alphabet.SetSelected(index: i, value: true);
        }
#endif
    }

    #region Overview Tab

    /// <summary>
    ///     Fills (and refills/adjusts) the cbx dropdown's text options
    /// </summary>
    /// <param name="filter"></param>
    private void FillCbx_Securities(string filter = "")
    {
        cbx_Securities.SelectedValue = string.Empty;
        List<SEDOL> sedolList = SEDOLs.Where(predicate: sedol =>
                                           ((sedol.Is_ISA_Compatible && ckb_ISAOnlySearch.Checked) ||
                                            !ckb_ISAOnlySearch.Checked) &&
                                           ((sedol.ETF_Type != Not_ETF_ETFType && ckb_ETFOnlySearch.Checked) ||
                                            !ckb_ETFOnlySearch.Checked) &&
                                           (string.IsNullOrWhiteSpace(value: filter) ||
                                            sedol.Name.Contains(value: filter, comparisonType: StringComparison.OrdinalIgnoreCase)))
                                      .ToList();

        // Create a BindingSource
        BindingSource bindingSource = [];

        // Assign the data source to the binding source
        bindingSource.DataSource = sedolList;

        // Set the ComboBox's DataSource property to the BindingSource
        cbx_Securities.DataSource = bindingSource;

        // Set the DisplayMember and ValueMember properties of the ComboBox
        cbx_Securities.DisplayMember = "Name";
        cbx_Securities.ValueMember = "SEDOL_ID";
    }

    #endregion

    #region Scrape & Parse

    /// <summary>
    ///     Pulls the FX info and deserialises the JSON output
    /// </summary>
    /// <param name="formInstance"></param>
    /// <param name="cancellationToken"></param>
    /// <returns></returns>
    private static async Task<bool> ReadJsonFXFromWebAsync(FrmMainApp formInstance,
        CancellationToken cancellationToken)
    {
        try
        {
            // Check if cancellation has been requested
            cancellationToken.ThrowIfCancellationRequested();

            string logMessageVal = "Reading FX Data From Web";
            AppendLogWindowText(tbx: formInstance.tbx_Log, appendText: logMessageVal,
                logMessageType: LogMessageTypes.Start);

            AppendLogWindowText(tbx: formInstance.tbx_Log,
                appendText: "Contacting FX Data website (http://www.floatrates.com/currency/gbp/)",
                logMessageType: LogMessageTypes.Info);


            HttpResponseMessage response =
                await HttpClient.GetAsync(requestUri: FxUrl, cancellationToken: cancellationToken);
            _ = response.EnsureSuccessStatusCode();

            string jsonString = await response.Content.ReadAsStringAsync(cancellationToken: cancellationToken);

            Dictionary<string, FXCurrency>? currencies =
                JsonConvert.DeserializeObject<Dictionary<string, FXCurrency>>(value: jsonString);

            Debug.Assert(condition: currencies != null, message: nameof(currencies) + " != null");
            foreach (KeyValuePair<string, FXCurrency> kvp in currencies)
            {
                FxCurrencies.Add(item: kvp.Value);
                AppendLogWindowText(tbx: formInstance.tbx_Log,
                    appendText: kvp.Value.Code + " " + kvp.Value.InverseRate);
            }

            AppendLogWindowText(tbx: formInstance.tbx_Log, appendText: logMessageVal,
                logMessageType: LogMessageTypes.Done);
            return true;
        }
        catch (Exception ex)
        {
            if (ex is OperationCanceledException or
                TaskCanceledException)
            {
                // Operation was cancelled
                AppendLogWindowText(tbx: formInstance.tbx_Log,
                    appendText: "Reading FX Data was cancelled. Process halted.",
                    logMessageType: LogMessageTypes.Info);
                return false;
            }

            AppendLogWindowText(tbx: formInstance.tbx_Log,
                appendText: "Failed to read or parse FX conversion values.",
                logMessageType: LogMessageTypes.Error);
            return false;
        }
    }

    /// <summary>
    ///     Loads up the URLs that belong to each letter of the alphabet
    /// </summary>
    /// <param name="formInstance"></param>
    /// <param name="cancellationToken"></param>
    /// <returns></returns>
    private static async Task CollateHLStocksByLetterAsync(FrmMainApp formInstance,
        CancellationToken cancellationToken)
    {
        const string logMessageVal = "Building Initial list of stocks and shares";
        AppendLogWindowText(tbx: formInstance.tbx_Log,
            appendText:
            $"{logMessageVal} - please note that it can appear that the app goes quiet here/at the beginning of any/each block, it shouldn't actually be dead so be patient.",
            logMessageType: LogMessageTypes.Start);

        foreach (char alphabetChar in selectedAlphabet)
        {
            AppendLogWindowText(tbx: formInstance.tbx_Log, appendText: "Parsing block: " + alphabetChar,
                logMessageType: LogMessageTypes.None);
            string respString;

            try
            {
                cancellationToken.ThrowIfCancellationRequested();

                string url = "https://www.hl.co.uk/shares/shares-search-results/" + alphabetChar;
                HttpResponseMessage response =
                    await HttpClient.GetAsync(requestUri: url, cancellationToken: cancellationToken);
                response.EnsureSuccessStatusCode();

                respString = await response.Content.ReadAsStringAsync(cancellationToken: cancellationToken);
            }
            catch (Exception ex)
            {
                if (ex is OperationCanceledException or TaskCanceledException)
                {
                    AppendLogWindowText(tbx: formInstance.tbx_Log,
                        appendText: "Parsing of alphabetical blocks was cancelled. Process halted.",
                        logMessageType: LogMessageTypes.Info);
                    return;
                }

                AppendLogWindowText(tbx: formInstance.tbx_Log, appendText: ex.Message,
                    logMessageType: LogMessageTypes.Error);
                return;
            }

            HtmlDocument doc = new();
            doc.LoadHtml(html: respString);

            HtmlNodeCollection? linkNodes = doc.DocumentNode.SelectNodes(xpath: "//a[@href]");

            if (linkNodes != null)
            {
                foreach (HtmlNode? linkNode in linkNodes)
                {
                    string href = linkNode.GetAttributeValue(name: "href", def: "");
#if DEBUG
                    //List<string> listOfURLsToDebug = new()
                    //{
                    //    "https://www.hl.co.uk/shares/shares-search-results/i/ishares-ii-plc-ftse-epranareit-asia-prop"
                    //};
                    List<string> listOfURLsToDebug = [];
                    if (href.Contains(value: "/shares/shares-search-results/" + alphabetChar + "/") &&
                        (listOfURLsToDebug.Any(predicate: href.Contains) ||
                         listOfURLsToDebug.Count == 0))
                    {
                        UrlListOfStocksAndShares.Add(item: href);
                    }
#else
                    if (href.Contains(value: "/shares/shares-search-results/" + alphabetChar + "/"))
                    {
                        UrlListOfStocksAndShares.Add(item: href);
                    }
#endif
                }
            }
        }

        AppendLogWindowText(tbx: formInstance.tbx_Log,
            appendText: "Collected " + UrlListOfStocksAndShares.Count + " items.",
            logMessageType: LogMessageTypes.None);

        await FireAndForgetAsync(urls: UrlListOfStocksAndShares, formInstance: formInstance,
            cancellationToken: cancellationToken);

        AppendLogWindowText(tbx: formInstance.tbx_Log, appendText: logMessageVal, logMessageType: LogMessageTypes.Done);
    }

    private static async Task FireAndForgetAsync(HashSet<string> urls, FrmMainApp formInstance,
        CancellationToken cancellationToken)
    {
        try
        {
            AppendLogWindowText(tbx: formInstance.tbx_Log, appendText: "Scraping all items.",
                logMessageType: LogMessageTypes.Start);

            // Each URL results in 2 requests (page + company info), hence the 2x. A sliding window of
            // requests (rather than fixed chunks) means one slow response doesn't stall the whole batch.
            // Runs on the thread pool, so HTML parsing/compression stays off the UI thread.
            ParallelOptions options = new()
            {
                MaxDegreeOfParallelism = Math.Max(1, _maxConnectionsPerServerSetting * 2),
                CancellationToken = cancellationToken
            };

            await Parallel.ForEachAsync(
                source: urls.SelectMany(url => new[] { url, url + "/company-information" }),
                parallelOptions: options,
                body: async (url, ct) => await GetHtmlAsync(url: url, formInstance: formInstance, cancellationToken: ct));

            AppendLogWindowText(tbx: formInstance.tbx_Log, appendText: "Scraping all items.",
                logMessageType: LogMessageTypes.Done);
        }
        catch (Exception ex) when (ex is OperationCanceledException or TaskCanceledException)
        {
            AppendLogWindowText(tbx: formInstance.tbx_Log,
                appendText: "Scraping of web pages was cancelled. Process halted.",
                logMessageType: LogMessageTypes.Info);
        }
    }

    private static async Task GetHtmlAsync(string url, FrmMainApp formInstance, CancellationToken cancellationToken)
    {
        try
        {
            HttpResponseMessage response = await HttpClient.GetAsync(requestUri: url, cancellationToken: cancellationToken);
            string htmlContent = await response.Content.ReadAsStringAsync(cancellationToken: cancellationToken);

            if (!url.Contains(value: "company-info"))
            {
                urlAndHtmlContentHashtable.AddOrUpdate(key: url,
                    value: HlPageParser.ReturnPageText(htmlContent: htmlContent));
            }
            else
            {
                urlAndCompanyInfoHashtable.AddOrUpdate(key: url,
                    value: HlPageParser.ReturnCompanyPageText(htmlContent: htmlContent));
            }

            IncrementCounterAndLogProgress(url: url, formInstance: formInstance, isSuccess: true);
        }
        catch (Exception ex) when (ex is not OperationCanceledException)
        {
            // one bad page shouldn't take down the whole scrape; CreateSEDOL skips items with missing data
            IncrementCounterAndLogProgress(url: url, formInstance: formInstance, isSuccess: false,
                errorMsg: ex.Message);
        }
    }

    /// <summary>
    ///     Creates the SEDOLs. Technically the primary key is the URL at the stage of creation.
    /// </summary>
    /// <param name="url"></param>
    /// <returns></returns>
    private static SEDOL? CreateSEDOL(string url)
    {
        if (urlAndHtmlContentHashtable.ContainsKey(key: url) &&
            urlAndCompanyInfoHashtable.ContainsKey(key: url + "/company-information"))
        {
            Application.DoEvents();

            Dictionary<string, string>? dataInContentHashtable = urlAndHtmlContentHashtable.Get(key: url);
            Dictionary<string, string>? dataInCompanyInfoHashtable = urlAndCompanyInfoHashtable.Get(key: url + "/company-information");
            if (dataInContentHashtable != null &&
                dataInCompanyInfoHashtable != null)
            {
                string logMessageVal = $"Parsing {url}";
                FrmMainApp frmMainAppInstance = (FrmMainApp)Application.OpenForms[name: "FrmMainApp"];

                if (dataInContentHashtable.Count == 0)
                {
                    return returnInvalidItem(logMessageVal, frmMainAppInstance);
                }

                string name = dataInContentHashtable["Name"];
                string sedolID = dataInContentHashtable["SEDOL"];
                string ticker = dataInContentHashtable["EPIC"];

                // exit if fails here
                if (string.IsNullOrWhiteSpace(value: sedolID) ||
                    string.IsNullOrWhiteSpace(value: ticker) ||
                    string.IsNullOrWhiteSpace(value: name) ||
                    name.Contains(value: "404"))
                {
                    return returnInvalidItem(logMessageVal, frmMainAppInstance);
                }

                string currISO3 = dataInContentHashtable["Currency"] ?? string.Empty;
                string currSign = TagsToModelValueTransformations.T2M_CurrencySign(currISO3: currISO3);
                double gbpEqivalent = TagsToModelValueTransformations.T2M_GBPEquivalent(currISO3: currISO3);

                double openVal = TagsToModelValueTransformations.T2M_Open_price(openStr: dataInContentHashtable["OpenPrice"] ?? string.Empty);
                double yearLow = TagsToModelValueTransformations.T2M_Year_low(yearLowStr: dataInContentHashtable["YearLow"] ?? string.Empty, openVal: openVal);
                double yearHigh = TagsToModelValueTransformations.T2M_Year_high(yearHighStr: dataInContentHashtable["YearHigh"] ?? string.Empty, openVal: openVal);
                double volume = TagsToModelValueTransformations.T2M_Volume(volumeStr: dataInContentHashtable["VolumeTraded"] ?? string.Empty);
                double marketCap = TagsToModelValueTransformations.T2M_Market_capitalisation(marketCapStr: dataInContentHashtable["MarketCap"] ?? string.Empty, currISO3: currISO3, currSign: currSign);

                string dividendYield = TagsToModelValueTransformations.T2M_Dividend_yield(divYieldStr: dataInContentHashtable["DividendYield"] ?? string.Empty);

                double peRatio = TagsToModelValueTransformations.T2M_PE_ratio(peRatioStr: dataInContentHashtable["PERatio"] ?? string.Empty, currSign: currSign);

                // this is a bit tricky. bonds & trusts don't _really_ have a corporate page so i have to trick around this stuff
                string type = dataInContentHashtable["Type"] ?? string.Empty;
                string sector = type.Contains("trust") ? "Trust" : type.Contains("bond") ? "Bond" : TagsToModelValueTransformations.T2M_Sector(dataInContentHashtableSector: dataInContentHashtable["Sector"] ?? string.Empty,
                        securityNameLowerCase: name, ticker: ticker, marketCapOverZero: marketCap > 0);

                string etfType = !sector.Contains(value: "ETF")
                    ? Not_ETF_ETFType
                    : TagsToModelValueTransformations.T2M_ETF_Type(name: name);

                _ = dataInCompanyInfoHashtable.TryGetValue("TopHoldings", out string top10_exposures);

                SEDOL newSedol = new()
                {
                    URL = url,
                    Name = name,
                    Is_ISA_Compatible = bool.Parse(dataInContentHashtable["IsIsaTradeable"]),
                    SEDOL_ID = sedolID,
                    Currency = currISO3,
                    Ticker = ticker,
                    Open_price = openVal,
                    Year_low = yearLow,
                    Year_high = yearHigh,
                    Volume = volume,
                    Dividend_yield = dividendYield,
                    PE_ratio = peRatio,
                    Market_capitalisation = marketCap,
                    GBP_Open = openVal * gbpEqivalent,
                    GBP_Year_low = yearLow * gbpEqivalent,
                    GBP_Year_high = yearHigh * gbpEqivalent,
                    GBP_Market_capitalisation = marketCap * gbpEqivalent,
                    Sector = sector,
                    ETF_Type = etfType,
                    Top10_Exposures = top10_exposures ?? string.Empty,
                };

                AppendLogWindowText(tbx: frmMainAppInstance.tbx_Log,
                    appendText: logMessageVal + $" - {name} - {sedolID}.",
                    logMessageType: LogMessageTypes.Done);

                return newSedol;
            }
        }
        else
        {
            return null;
        }

        return null;

        static SEDOL? returnInvalidItem(string logMessageVal, FrmMainApp frmMainAppInstance)
        {
            AppendLogWindowText(tbx: frmMainAppInstance.tbx_Log, appendText: logMessageVal + " - invalid item.",
                                    logMessageType: LogMessageTypes.Error);
            return null;
        }
    }

    #endregion

    #region Events

    #region Generic

    /// <summary>
    ///     Fires the process to save the findings to a CSV
    /// </summary>
    /// <param name="sender"></param>
    /// <param name="e"></param>
    private void btn_SaveToCSV_Click(object sender, EventArgs e)
    {
        WriteOutputToCSV();
    }

    /// <summary>
    ///     Reads the CSV file for ETF categories
    /// </summary>
    /// <param name="sender"></param>
    /// <param name="e"></param>
    private void btn_ReloadCategories_Click(object sender, EventArgs e)
    {
        GetETFTypesFromCSV();
        _ = MessageBox.Show(text: "Categories reloaded.");
        foreach (SEDOL sedol in SEDOLs)
        {
            string name = sedol.Name;
            sedol.ETF_Type = !sedol.Sector.Contains(value: "ETF")
                ? Not_ETF_ETFType
                : TagsToModelValueTransformations.T2M_ETF_Type(name: name);
        }
    }

    /// <summary>
    ///     Loads the About Form
    /// </summary>
    /// <param name="sender"></param>
    /// <param name="e"></param>
    private void tsm_About_Click(object sender, EventArgs e)
    {
        FrmAboutBox frmAboutBox = new();
        _ = frmAboutBox.ShowDialog();
    }

    /// <summary>
    ///     Open the ETF_Types.csv in the external editor
    /// </summary>
    /// <param name="sender"></param>
    /// <param name="e"></param>
    private void tsmi_EditCategorisationCSV_Click(object sender, EventArgs e)
    {
        // copy the Example file if "normal" doesn't exist.
        if (!File.Exists(path: Path.Combine(path1: HelperVariables.GetResourcesFolderString(), path2: "ETF_Types.csv")))
        {
            File.Copy(
                sourceFileName: Path.Combine(path1: HelperVariables.GetResourcesFolderString(),
                    path2: "ETF_Types_Example.csv"),
                destFileName: Path.Combine(path1: HelperVariables.GetResourcesFolderString(), path2: "ETF_Types.csv"));
        }

        _ = new Process
        {
            StartInfo = new ProcessStartInfo(fileName: Path.Combine(path1: HelperVariables.GetResourcesFolderString(),
                path2: "ETF_Types.csv"))
            {
                UseShellExecute = true
            }
        }.Start();
    }


    private void tsmi_DarkishMode_Click(object sender, EventArgs e)
    {
        HelperVariables.UserSettingUseDarkMode = tsmi_DarkishMode.Checked;
        SetAppTheme();

        HelperDataApplicationSettings.DataWriteSQLiteSettings(tableName: "settings", settingId: "Theme",
            settingValue: HelperVariables.UserSettingUseDarkMode ? "Dark" : "Light");
    }

    private void FrmMainApp_FormClosing(object sender, FormClosingEventArgs e)
    {
        HelperDataApplicationSettings.DataDeleteSQLitesettingsCleanup();
        HelperDataApplicationSettings.DataVacuumDatabase();
    }

    #endregion

    #region tpg_ScrapeMainGrid

    /// <summary>
    ///     Commences the scrape
    /// </summary>
    /// <param name="sender"></param>
    /// <param name="e"></param>
    private async void btn_StartScrape_Click(object sender, EventArgs e)
    {
        selectedAlphabet = lbx_Alphabet.SelectedItems.Cast<char>().ToArray();
        if (selectedAlphabet.Length > 0)
        {
            // reset things.
            btn_StartScrape.Enabled = false;
            btn_Stop.Enabled = true;
            btn_ReloadCategories.Enabled = false;
            btn_SaveToCSV.Enabled = false;
            tpg_Overview.Enabled = false;

            tbx_Log.Clear();
            FxCurrencies.Clear();
            urlAndHtmlContentHashtable.Clear();
            urlAndCompanyInfoHashtable.Clear();
            UrlListOfStocksAndShares.Clear();

            _urlCounter = 0;

            cancellationTokenSource = new CancellationTokenSource();

            if (await ReadJsonFXFromWebAsync(formInstance: this, cancellationToken: cancellationTokenSource.Token))
            {
                await CollateHLStocksByLetterAsync(formInstance: this,
                    cancellationToken: cancellationTokenSource.Token);
            }

            foreach (SEDOL? newSedol in UrlListOfStocksAndShares.Select(selector: CreateSEDOL).OfType<SEDOL>())
            {
                if (!string.IsNullOrWhiteSpace(value: newSedol?.SEDOL_ID))
                {
                    _ = SEDOLs.Add(item: newSedol);
                }
            }

            btn_StartScrape.Enabled = true;
            btn_Stop.Enabled = false;
            btn_ReloadCategories.Enabled = true;
            if (SEDOLs.Count > 0)
            {
                btn_SaveToCSV.Enabled = true;
                tpg_Overview.Enabled = true;
                FrmMainApp frmMainAppInstance = (FrmMainApp)Application.OpenForms[name: "FrmMainApp"];
                string logMessageVal = "Building parsing data - ready for output.";
                AppendLogWindowText(tbx: frmMainAppInstance.tbx_Log, appendText: logMessageVal,
                    logMessageType: LogMessageTypes.Done);
                nic_ProcessFinished.ShowBalloonTip(timeout: 150);
            }
            else
            {
                FrmMainApp frmMainAppInstance = (FrmMainApp)Application.OpenForms[name: "FrmMainApp"];
                string logMessageVal = "Building parsing data encountered an error. No data was parsed.";
                AppendLogWindowText(tbx: frmMainAppInstance.tbx_Log, appendText: logMessageVal,
                    logMessageType: LogMessageTypes.Error);
            }

            // release memory
            urlAndHtmlContentHashtable.Clear();
            urlAndCompanyInfoHashtable.Clear();
        }
        else
        {
            _ = MessageBox.Show(text: "You do need to select at least one character you know.",
                caption: "Nothing selected.",
                buttons: MessageBoxButtons.OK, icon: MessageBoxIcon.Warning);
        }
    }


    /// <summary>
    ///     Sends a CT to the ongoing process
    /// </summary>
    /// <param name="sender"></param>
    /// <param name="e"></param>
    private void btn_Stop_Click(object sender, EventArgs e)
    {
        // Check if cancellationTokenSource is initialized and if there's an ongoing task
        if (cancellationTokenSource != null &&
            !cancellationTokenSource.IsCancellationRequested)
        {
            // Cancel the ongoing tasks
            cancellationTokenSource.Cancel();
        }

        btn_StartScrape.Enabled = true;
        btn_Stop.Enabled = false;
        btn_ReloadCategories.Enabled = false;
        btn_SaveToCSV.Enabled = false;
    }

    #endregion

    #region tpg_Overview

    private void tpg_Overview_Enter(object sender, EventArgs e)
    {
        FillCbx_Securities();
    }

    private void tbx_Search_TextChanged(object sender, EventArgs e)
    {
        FillCbx_Securities(filter: tbx_Search.Text);
    }

    private void ckb_ISAOnlySearch_CheckedChanged(object sender, EventArgs e)
    {
        FillCbx_Securities(filter: tbx_Search.Text);
    }

    private void ckb_ETFOnlySearch_CheckedChanged(object sender, EventArgs e)
    {
        FillCbx_Securities(filter: tbx_Search.Text);
    }

    private void cbx_Securities_SelectedValueChanged(object sender, EventArgs e)
    {
        if (cbx_Securities.SelectedItem is SEDOL selectedSEDOL)
        {
            // Access the value member (SEDOLID) of the selected item
            string selectedSedolSedolId = selectedSEDOL.SEDOL_ID;

            // Find the SEDOL object in sedols where SEDOLID matches x
            SEDOL sedol = SEDOLs.FirstOrDefault(predicate: s => s.SEDOL_ID == selectedSedolSedolId);

            if (sedol != null)
            {
                tbx_Name.Text = sedol.Name;
                tbx_Ticker.Text = sedol.Ticker;
                tbx_SEDOL.Text = sedol.SEDOL_ID;
                ckb_ISA.Checked = sedol.Is_ISA_Compatible;
                ckb_ISA.Text = $"{(sedol.Is_ISA_Compatible ? "Can" : "Can't")} be held in an ISA";
                llb_URL.Links.Clear();
                _ = llb_URL.Links.Add(start: 0, length: 3, linkData: sedol.URL);
                tbx_Sector.Text = sedol.Sector;
                tbx_ETFType.Text = sedol.ETF_Type;
                tbx_Currency.Text = sedol.Currency;
                tbx_DivYield.Text = sedol.Dividend_yield;
                tbx_OpenPrice_OC.Text = sedol.Open_price.ToString(format: "N4", provider: NumberFormatInfo.CurrentInfo);
                tbx_OpenPrice_GBP.Text =
                    sedol.GBP_Open.ToString(format: "C4", provider: new CultureInfo(name: "en-GB"));
                tbx_YearLow_OC.Text = sedol.Year_low.ToString(format: "N4", provider: NumberFormatInfo.CurrentInfo);
                tbx_YearLow_GBP.Text =
                    sedol.GBP_Year_low.ToString(format: "C4", provider: new CultureInfo(name: "en-GB"));
                tbx_YearHigh_OC.Text = sedol.Year_high.ToString(format: "N4", provider: NumberFormatInfo.CurrentInfo);
                tbx_YearHigh_GBP.Text =
                    sedol.GBP_Year_high.ToString(format: "C4", provider: new CultureInfo(name: "en-GB"));
                tbx_MarketCap_OC.Text =
                    sedol.Market_capitalisation.ToString(format: "N0", provider: NumberFormatInfo.CurrentInfo);
                tbx_MarketCap_GBP.Text =
                    sedol.GBP_Market_capitalisation.ToString(format: "C0", provider: new CultureInfo(name: "en-GB"));
                tbx_PE.Text = sedol.PE_ratio.ToString(format: "N4", provider: NumberFormatInfo.CurrentInfo);
                tbx_Volume.Text = sedol.Volume.ToString(format: "N0", provider: NumberFormatInfo.CurrentInfo);
            }
        }
    }

    /// <summary>
    /// Opens the browser to the link clicked
    /// </summary>
    /// <param name="sender"></param>
    /// <param name="e"></param>
    private void llb_URL_LinkClicked(object sender, LinkLabelLinkClickedEventArgs e)
    {
        if (e.Link != null)
        {
            _ = Process.Start(startInfo: new ProcessStartInfo(fileName: e.Link.LinkData.ToString())
            { UseShellExecute = true });
        }
    }

    #endregion

    #endregion

    #region Utils

    /// <summary>
    ///     Puts text into the log window (append)
    /// </summary>
    /// <param name="tbx"></param>
    /// <param name="appendText"></param>
    /// <param name="logMessageType"></param>
    private static void AppendLogWindowText(TextBox tbx, string appendText,
        LogMessageTypes logMessageType = LogMessageTypes.None)
    {
        string messageType = logMessageType != LogMessageTypes.None ? "[" + logMessageType + "] " : "";

        if (tbx.InvokeRequired)
        {
            _ = tbx.Invoke(method: (MethodInvoker)delegate
            {
                tbx.AppendText(text: DateTime.Now + " - " + messageType + appendText +
                                     Environment.NewLine);
                tbx.SelectionStart = tbx.Text.Length;
                tbx.ScrollToCaret();
            });
        }
        else
        {
            tbx.AppendText(text: DateTime.Now + " - " + messageType + appendText +
                                 Environment.NewLine);
            tbx.SelectionStart = tbx.Text.Length;
            tbx.ScrollToCaret();
        }
    }


    /// <summary>
    ///     Logs the count of URLs parsed and returns the result message accordingly
    /// </summary>
    /// <param name="url"></param>
    /// <param name="formInstance"></param>
    /// <param name="isSuccess"></param>
    /// <param name="errorMsg"></param>
    private static void IncrementCounterAndLogProgress(string url, FrmMainApp formInstance, bool isSuccess,
        string errorMsg = "")
    {
        lock (urlCounterLock)
        {
            _urlCounter++;
            if (isSuccess)
            {
                AppendLogWindowText(tbx: formInstance.tbx_Log,
                    appendText: $"{_urlCounter} of {UrlListOfStocksAndShares.Count * 2} items scraped - {url}.",
                    logMessageType: LogMessageTypes.None);
            }
            else
            {
                AppendLogWindowText(tbx: formInstance.tbx_Log,
                    appendText: $"Error fetching URL '{url}': {errorMsg}",
                    logMessageType: LogMessageTypes.Error);
            }
        }
    }


    /// <summary>
    ///     Sets the application theme at startup based on the user's settings.
    /// </summary>
    /// <remarks>
    ///     If the user has chosen to use dark mode, the method sets the theme color to dark and applies a custom renderer to
    ///     the menu strip.
    ///     If the user has not chosen to use dark mode, the method sets the theme color to light and uses the default
    ///     rendering for the controls.
    /// </remarks>
    private void SetAppTheme()
    {
        // the custom logic is ugly af so no need to be pushy about it in light mode.
        if (!HelperVariables.UserSettingUseDarkMode)
        {
            tcr_Main.DrawMode = TabDrawMode.Normal;
            mns_Main.RenderMode = ToolStripRenderMode.System;
        }
        else
        {
            mns_Main.Renderer = new DarkMenuStripRenderer();
        }

        // adds colour/theme
        _ = new WinFormsDarkThemerNinja.Themer();
        Themer.DarkColor = ColorTranslator.FromHtml("#2B2D31");
        Themer.ApplyThemeToControl(
            control: this,
            themeStyle: HelperVariables.UserSettingUseDarkMode ?
            Themer.ThemeStyle.Custom :
            Themer.ThemeStyle.Default
            );
    }


    // via https://stackoverflow.com/questions/9260303/how-to-change-menu-hover-color
    private class DarkMenuStripRenderer : ToolStripProfessionalRenderer
    {
        public DarkMenuStripRenderer() : base(professionalColorTable: new DarkColours())
        {
        }

        protected override void OnRenderItemText(ToolStripItemTextRenderEventArgs e)
        {
            e.TextColor = Color.White;

            base.OnRenderItemText(e: e);
        }
    }


    private class DarkColours : ProfessionalColorTable
    {
        public override Color MenuItemBorder =>
            ColorTranslator.FromHtml(htmlColor: "#BAB9B9");

        public override Color MenuBorder =>
            Color.Silver; //added for changing the menu border

        public override Color MenuItemPressedGradientBegin =>
            ColorTranslator.FromHtml(htmlColor: "#4C4A48");

        public override Color MenuItemPressedGradientEnd =>
            ColorTranslator.FromHtml(htmlColor: "#5F5D5B");

        public override Color ToolStripBorder =>
            ColorTranslator.FromHtml(htmlColor: "#4C4A48");

        public override Color MenuItemSelectedGradientBegin =>
            ColorTranslator.FromHtml(htmlColor: "#4C4A48");

        public override Color MenuItemSelectedGradientEnd =>
            ColorTranslator.FromHtml(htmlColor: "#5F5D5B");

        public override Color MenuItemSelected =>
            ColorTranslator.FromHtml(htmlColor: "#5F5D5B");

        public override Color ToolStripDropDownBackground =>
            ColorTranslator.FromHtml(htmlColor: "#404040");

        public override Color ToolStripGradientBegin =>
            ColorTranslator.FromHtml(htmlColor: "#404040");

        public override Color ToolStripGradientEnd =>
            ColorTranslator.FromHtml(htmlColor: "#404040");

        public override Color ToolStripGradientMiddle =>
            ColorTranslator.FromHtml(htmlColor: "#404040");

        public override Color ImageMarginGradientBegin =>
            ColorTranslator.FromHtml(htmlColor: "#404040");

        public override Color ImageMarginGradientEnd =>
            ColorTranslator.FromHtml(htmlColor: "#404040");

        public override Color ImageMarginGradientMiddle =>
            ColorTranslator.FromHtml(htmlColor: "#404040");

        public override Color ImageMarginRevealedGradientBegin =>
            ColorTranslator.FromHtml(htmlColor: "#404040");

        public override Color ImageMarginRevealedGradientEnd =>
            ColorTranslator.FromHtml(htmlColor: "#404040");

        public override Color ImageMarginRevealedGradientMiddle =>
            ColorTranslator.FromHtml(htmlColor: "#404040");
    }

    #endregion

    #region CSV

    /// <summary>
    ///     Loads the ETF_Types from the CSV file
    /// </summary>
    private static void GetETFTypesFromCSV()
    {
        string resourcesFolder = Path.Combine(path1: AppDomain.CurrentDomain.BaseDirectory,
            path2: "Resources");
        string ETF_TypesCSVPath = File.Exists(path: Path.Combine(path1: resourcesFolder, path2: "ETF_Types.csv"))
            ? Path.Combine(path1: resourcesFolder, path2: "ETF_Types.csv")
            : Path.Combine(path1: resourcesFolder, path2: "ETF_Types_Example.csv");

        using FileStream stream = new(ETF_TypesCSVPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
        using StreamReader reader = new(stream);
        using CsvReader csv = new(reader, CultureInfo.InvariantCulture);
        ETF_Types = csv.GetRecords<ETFType>().ToList();
    }

    /// <summary>
    ///     Writes the scraping output to a CSV file in the Output folder
    /// </summary>
    private void WriteOutputToCSV()
    {
        string outputFolder = Path.Combine(path1: AppDomain.CurrentDomain.BaseDirectory,
            path2: "Output");
        _ = Directory.CreateDirectory(path: outputFolder);

        string nowDTStr = DateTime.Now.ToString(format: "yyyyMMdd_HHmmss");

        // if filters are active ask user if they want to use selection
        DialogResult dialogResult = DialogResult.No;
        if (cbx_Securities.Items.Count > 0 &&
            cbx_Securities.Items.Count < SEDOLs.Count &&
            tcr_Main.SelectedTab == tpg_Overview)
        {
            dialogResult = MessageBox.Show(text: "Only export filtered items?",
                caption: "Filters are active.",
                buttons: MessageBoxButtons.YesNoCancel, icon: MessageBoxIcon.Question);
        }

        List<SEDOL> sedolList = dialogResult == DialogResult.Yes
            ? cbx_Securities.Items.OfType<SEDOL>().ToList()
            : dialogResult == DialogResult.No ? [.. SEDOLs] : [];

        if (dialogResult != DialogResult.Cancel)
        {
            try
            {
                using StreamWriter writer =
                    new(path: Path.Combine(path1: outputFolder, path2: $"HLWebScraper_Output_{nowDTStr}.csv"));
                using CsvWriter csv = new(writer: writer, culture: CultureInfo.InvariantCulture);
                csv.WriteHeader<SEDOL>();
                csv.NextRecord();
                foreach (SEDOL record in sedolList)
                {
                    csv.WriteRecord(record: record);
                    csv.NextRecord();
                }

                writer.Flush();
                _ = MessageBox.Show(text: $"Finished writing the CSV file - HLWebScraper_Output_{nowDTStr}.csv",
                    caption: "Ok writing CSV",
                    buttons: MessageBoxButtons.OK, icon: MessageBoxIcon.Information);
            }
            catch (Exception ex)
            {
                _ = MessageBox.Show(text: $"Failed to write the CSV file - {ex.Message}", caption: "Error writing CSV",
                    buttons: MessageBoxButtons.OK,
                    icon: MessageBoxIcon.Error);
            }
        }
    }

    #endregion


    /// <summary>
    /// Select all items in the listbox
    /// </summary>
    /// <param name="sender"></param>
    /// <param name="e"></param>
    private void btn_All_Click(object sender, EventArgs e)
    {
        // Select All
        for (int i = 0; i < lbx_Alphabet.Items.Count; i++)
        {
            lbx_Alphabet.SetSelected(i, true);
        }
    }

    /// <summary>
    /// Select no items in the listbox
    /// </summary>
    /// <param name="sender"></param>
    /// <param name="e"></param>
    private void btn_None_Click(object sender, EventArgs e)
    {
        lbx_Alphabet.ClearSelected();
    }

    internal enum LogMessageTypes
    {
        Start,
        Done,
        Error,
        Info,
        None
    }
}