From 9d0015bfcc244265b4296c45c333504be5b94736 Mon Sep 17 00:00:00 2001 From: Gabriella Casap Date: Fri, 26 Sep 2025 18:53:26 +0100 Subject: [PATCH] fix search-engine not allowing arson's CA --- search-engine/src/arson-ca.crt | 28 ++++++++++++++++++++++++++++ search-engine/src/crawler.rs | 4 +++- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 search-engine/src/arson-ca.crt diff --git a/search-engine/src/arson-ca.crt b/search-engine/src/arson-ca.crt new file mode 100644 index 0000000..01ad78d --- /dev/null +++ b/search-engine/src/arson-ca.crt @@ -0,0 +1,28 @@ +-----BEGIN CERTIFICATE----- +MIIEvjCCAyagAwIBAgIRAOgNdrOTI5GLs9YoNMGXILkwDQYJKoZIhvcNAQELBQAw +dzEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMSYwJAYDVQQLDB1zbWFy +dGNvZGVyQG5peG9zIChTbWFydGNvZGVyKTEtMCsGA1UEAwwkbWtjZXJ0IHNtYXJ0 +Y29kZXJAbml4b3MgKFNtYXJ0Y29kZXIpMB4XDTI1MDkwODA4MDIxN1oXDTM1MDkw +ODA4MDIxN1owdzEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMSYwJAYD +VQQLDB1zbWFydGNvZGVyQG5peG9zIChTbWFydGNvZGVyKTEtMCsGA1UEAwwkbWtj +ZXJ0IHNtYXJ0Y29kZXJAbml4b3MgKFNtYXJ0Y29kZXIpMIIBojANBgkqhkiG9w0B +AQEFAAOCAY8AMIIBigKCAYEAqkhX/N1Mjzoq9CgdREOFkW+TPeo5yshLHVFqWiY/ +8gxmuFd6kK+TfnsUKeDKK2z/KdZWlWjSxFdvZSn19TPsZ1OLgn0SbWXvUiFjjlJk +/bgVU7bwAR+TvEOMJ/TTK/T9LkWLK4pQov5+LtnuLh5s0aLUW/eV1OcYNdQGuEhD +IzN8ITp7vZeKTAB0TGu+hVG+xsVdnRPsmugP0EQy7jBtB4KjK+CCxkMYQRS17h22 +RyDDwvtLIGWWHfza6M1MkEqfTevzBR/3fAt4kFKt19p+pVG7bHuVxHaVdiMLPq0h +vR52ELxNb3v8j7a0ZTQG3sym+1J0Avr5z5onuPx0rxsaLoX7NvzPQZk+hxEYynOP +f0lgoMOJEcMPXcj+dEBpB5Q5igR9OEs53wINYvN5lOw6X466DZH8ofr/h8uAtYBp +y9DXdbOddTFSIHfYDlQsdK8txgrW0kgs1raoS8h2EI9CETCjKzeuID6bp1/6K96n +G5pk5aat+ElBkWa4o4OCiRWRAgMBAAGjRTBDMA4GA1UdDwEB/wQEAwICBDASBgNV +HRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBQysYpcKzkvcIPrQQDo/olzWnKGoTAN +BgkqhkiG9w0BAQsFAAOCAYEAcjstUc9k2kgwodhyh+R35p+SZRDGuWgBZVn7wHFS +4W7TuSyOw7RLHlGOtJ7SxzqKoOQmMaC0IhFN50znE9FocjcHOXMcRQpc/ql18Ohh +hVIqli5GnR3N+JRWGo234BZU73SPkskQnP4xw6gZKwwTiMjMltuY5KuRuCQpk4/w +Avn9k8fgrFidYu6PAYzx+r/JyVYISzLLoowgK8hK4bNWIujESvF87NxcNGwV/+j/ +q1sI+yKplw35Jjvhg0UBNrRAbujBjIthL6rPLrqk1e/modoOE2shT/QelmkJUtOZ +aAZVMSQfKn6zsqypHqeBdJg8djDT9YqQZDu2l6yOQSiGb81pJxHyPYso17JkjvKA +SyluR0RtiTeib7VQCVIHjgcfFyQP1jBELk2Yq9HeYMg89M1U1AThI4JMCA6ukYVR +oktSBQJdPEQYu1Geve/UU04g5JzstBMrET9EsRlyfg2/B3o2/TbZIlRPbAIRdVhn +XsC+LWBb5waKWhq6Ti+6CNuv +-----END CERTIFICATE----- diff --git a/search-engine/src/crawler.rs b/search-engine/src/crawler.rs index fc19e8b..916985c 100644 --- a/search-engine/src/crawler.rs +++ b/search-engine/src/crawler.rs @@ -10,6 +10,8 @@ use url::Url; use crate::config::Config; use crate::models::{Domain, DomainRepository, CrawledPage}; +static ARSON_CA: &str = include_str!("arson-ca.crt"); + #[derive(Debug, Clone)] struct CrawledPageWithHtml { crawled_page: CrawledPage, @@ -35,7 +37,7 @@ impl DomainCrawler { request_timeout: config.crawler_timeout(), user_agent: config.search.crawler_user_agent.clone(), max_redirects: config.crawler.max_redirects, - custom_ca_certificates: vec![ca_cert], + custom_ca_certificates: vec![ca_cert, ARSON_CA.to_string()], ..Default::default() };