Merge pull request #119 from someoneidoknow/search-engine-fix

fix search-engine not allowing arson's CA

search-engine/src/arson-ca.crt

@@ -0,0 +1,28 @@
+-----BEGIN CERTIFICATE-----
+MIIEvjCCAyagAwIBAgIRAOgNdrOTI5GLs9YoNMGXILkwDQYJKoZIhvcNAQELBQAw
+dzEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMSYwJAYDVQQLDB1zbWFy
+dGNvZGVyQG5peG9zIChTbWFydGNvZGVyKTEtMCsGA1UEAwwkbWtjZXJ0IHNtYXJ0
+Y29kZXJAbml4b3MgKFNtYXJ0Y29kZXIpMB4XDTI1MDkwODA4MDIxN1oXDTM1MDkw
+ODA4MDIxN1owdzEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMSYwJAYD
+VQQLDB1zbWFydGNvZGVyQG5peG9zIChTbWFydGNvZGVyKTEtMCsGA1UEAwwkbWtj
+ZXJ0IHNtYXJ0Y29kZXJAbml4b3MgKFNtYXJ0Y29kZXIpMIIBojANBgkqhkiG9w0B
+AQEFAAOCAY8AMIIBigKCAYEAqkhX/N1Mjzoq9CgdREOFkW+TPeo5yshLHVFqWiY/
+8gxmuFd6kK+TfnsUKeDKK2z/KdZWlWjSxFdvZSn19TPsZ1OLgn0SbWXvUiFjjlJk
+/bgVU7bwAR+TvEOMJ/TTK/T9LkWLK4pQov5+LtnuLh5s0aLUW/eV1OcYNdQGuEhD
+IzN8ITp7vZeKTAB0TGu+hVG+xsVdnRPsmugP0EQy7jBtB4KjK+CCxkMYQRS17h22
+RyDDwvtLIGWWHfza6M1MkEqfTevzBR/3fAt4kFKt19p+pVG7bHuVxHaVdiMLPq0h
+vR52ELxNb3v8j7a0ZTQG3sym+1J0Avr5z5onuPx0rxsaLoX7NvzPQZk+hxEYynOP
+f0lgoMOJEcMPXcj+dEBpB5Q5igR9OEs53wINYvN5lOw6X466DZH8ofr/h8uAtYBp
+y9DXdbOddTFSIHfYDlQsdK8txgrW0kgs1raoS8h2EI9CETCjKzeuID6bp1/6K96n
+G5pk5aat+ElBkWa4o4OCiRWRAgMBAAGjRTBDMA4GA1UdDwEB/wQEAwICBDASBgNV
+HRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBQysYpcKzkvcIPrQQDo/olzWnKGoTAN
+BgkqhkiG9w0BAQsFAAOCAYEAcjstUc9k2kgwodhyh+R35p+SZRDGuWgBZVn7wHFS
+4W7TuSyOw7RLHlGOtJ7SxzqKoOQmMaC0IhFN50znE9FocjcHOXMcRQpc/ql18Ohh
+hVIqli5GnR3N+JRWGo234BZU73SPkskQnP4xw6gZKwwTiMjMltuY5KuRuCQpk4/w
+Avn9k8fgrFidYu6PAYzx+r/JyVYISzLLoowgK8hK4bNWIujESvF87NxcNGwV/+j/
+q1sI+yKplw35Jjvhg0UBNrRAbujBjIthL6rPLrqk1e/modoOE2shT/QelmkJUtOZ
+aAZVMSQfKn6zsqypHqeBdJg8djDT9YqQZDu2l6yOQSiGb81pJxHyPYso17JkjvKA
+SyluR0RtiTeib7VQCVIHjgcfFyQP1jBELk2Yq9HeYMg89M1U1AThI4JMCA6ukYVR
+oktSBQJdPEQYu1Geve/UU04g5JzstBMrET9EsRlyfg2/B3o2/TbZIlRPbAIRdVhn
+XsC+LWBb5waKWhq6Ti+6CNuv
+-----END CERTIFICATE-----

search-engine/src/crawler.rs

@@ -10,6 +10,8 @@ use url::Url;
 use crate::config::Config;
 use crate::models::{Domain, DomainRepository, CrawledPage};
 
+static ARSON_CA: &str = include_str!("arson-ca.crt");
+
 #[derive(Debug, Clone)]
 struct CrawledPageWithHtml {
     crawled_page: CrawledPage,
@@ -35,7 +37,7 @@ impl DomainCrawler {
             request_timeout: config.crawler_timeout(),
             user_agent: config.search.crawler_user_agent.clone(),
             max_redirects: config.crawler.max_redirects,
-            custom_ca_certificates: vec![ca_cert],
+            custom_ca_certificates: vec![ca_cert, ARSON_CA.to_string()],
             ..Default::default()
         };