diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..ae1f296 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# split manifest의 SHA-256은 artifact 재현성 검증에 쓰이므로 +# 체크아웃 플랫폼과 무관하게 항상 LF로 고정한다. +# (core.autocrlf=true 환경에서 CRLF로 변환되면 같은 split인데도 해시가 달라진다) +data_science/SMSModel/splits/*.csv text eol=lf diff --git a/app/analysis/institution/analyzer.py b/app/analysis/institution/analyzer.py index 5c18ef3..13d5c38 100644 --- a/app/analysis/institution/analyzer.py +++ b/app/analysis/institution/analyzer.py @@ -43,10 +43,7 @@ def _find_mentioned_institution(text: str) -> OfficialInstitution | None: return None -# 문자에 언급된 기관명과 실제 링크된 URL의 도메인이 그 기관의 공식 도메인과 일치하는지 대조. -# 기관 언급 자체가 없으면 대조할 대상이 없으므로 checked=False로 스킵하지만, URL이 없어서 -# 도메인 대조를 못 한 경우에도 감지된 기관명은 채워서 반환한다 - 발신번호-기관 연계(issue #67) -# 등 URL 유무와 무관하게 "이 문자가 어느 기관 명의였는지"가 필요한 소비자를 위함이다. +# 문자에 언급된 기관명과 실제 링크된 URL의 도메인이 그 기관의 공식 도메인과 일치하는지 대조 def analyze_institution_match(text: str, traced_url: str | None = None) -> dict: no_institution_result = { "checked": False, diff --git a/app/analysis/institution/registry.py b/app/analysis/institution/registry.py index 40c8348..5d81aea 100644 --- a/app/analysis/institution/registry.py +++ b/app/analysis/institution/registry.py @@ -1,18 +1,11 @@ from dataclasses import dataclass, field - @dataclass(frozen=True) class OfficialInstitution: - # 공식 기관명 (로그/응답 표시용) name: str - # 문자에서 실제 쓰이는 표기 변형 (이 중 하나라도 본문에 있으면 해당 기관 언급으로 간주) aliases: tuple[str, ...] - # 공식 도메인. 서브도메인도 인정하려면 상위 도메인만 등록 (예: "kbstar.com"이면 "obank.kbstar.com"도 매치) official_domains: tuple[str, ...] - # 공식 대표번호(고객센터). 각 기관 공식 홈페이지에서 직접 확인해 채움 (아래 출처 주석 참고) official_phone_numbers: tuple[str, ...] = field(default_factory=tuple) - # SMS 실제 발신번호는 기관마다 여러 개이고 수시로 바뀌며 공개된 단일 출처가 없어 - # 신뢰할 만한 데이터를 확보하기 전까지 비워둔다 — 향후 실제 수신 데이터가 쌓이면 채울 것. known_sender_numbers: tuple[str, ...] = field(default_factory=tuple) @@ -22,6 +15,7 @@ class OfficialInstitution: # 잘못된 공식 정보를 노출하면 오탐/오정보 제공으로 이어지므로, 향후 정보 추가/변경 시에도 # 반드시 공식 채널에서 직접 재확인할 것. OFFICIAL_INSTITUTIONS: tuple[OfficialInstitution, ...] = ( + # 은행 OfficialInstitution( "국민은행", ("국민은행", "KB국민은행"), ("kbstar.com",), ("1588-9999", "1599-9999", "1644-9999") @@ -43,21 +37,22 @@ class OfficialInstitution: OfficialInstitution( "우체국", ("우체국", "우정사업본부"), ("epost.go.kr", "koreapost.go.kr"), ("1588-1300",) ), + # 카드사 OfficialInstitution("신한카드", ("신한카드",), ("shinhancard.com",), ("1544-7000",)), OfficialInstitution("삼성카드", ("삼성카드",), ("samsungcard.com",), ("1588-8700",)), OfficialInstitution("현대카드", ("현대카드",), ("hyundaicard.com",), ("1577-6000",)), OfficialInstitution("KB국민카드", ("KB국민카드", "국민카드"), ("kbcard.com",), ("1588-1688",)), OfficialInstitution("롯데카드", ("롯데카드",), ("lottecard.co.kr",), ("1588-8100",)), - # 최초 작성 시 "uricard.com"으로 잘못 기재했던 것을 검증 과정에서 발견해 수정함 OfficialInstitution("우리카드", ("우리카드",), ("wooricard.com",), ("1588-9955", "1599-9955")), + # 공공/사법기관 OfficialInstitution("금융감독원", ("금융감독원",), ("fss.or.kr",), ("1332",)), OfficialInstitution("금융위원회", ("금융위원회",), ("fsc.go.kr",), ("02-2100-2500",)), OfficialInstitution("경찰청", ("경찰청",), ("police.go.kr",), ("182",)), OfficialInstitution("검찰청", ("검찰청",), ("spo.go.kr",), ("1301",)), - # 홈택스(hometax.go.kr)는 국세청이 직접 운영하는 전자세정 서비스 도메인이라 함께 등록 — - # 실제로 nts.go.kr보다 홈택스 링크가 더 흔히 쓰이므로 빠지면 정상 링크가 오탐됨 + + # 홈택스(hometax.go.kr)는 국세청이 직접 운영하는 전자세정 서비스 도메인이라 함께 등록 OfficialInstitution("국세청", ("국세청",), ("nts.go.kr", "hometax.go.kr"), ("126",)), OfficialInstitution("관세청", ("관세청",), ("customs.go.kr",), ("125",)), OfficialInstitution( diff --git a/app/analysis/text/structural_features.py b/app/analysis/text/structural_features.py index 29d5887..ec623b2 100644 --- a/app/analysis/text/structural_features.py +++ b/app/analysis/text/structural_features.py @@ -49,7 +49,20 @@ ) -# 배열의 열 순서가 학습 및 추론에서 동일해야 하므로 상수로 고정합니다. +# 합법 광고 문자의 법정 표기 +AD_DISCLOSURE_PATTERN = re.compile( + r"\(\s*광고\s*\)|\[\s*광고\s*\]|^광고", + re.IGNORECASE | re.MULTILINE, +) + +OPT_OUT_PATTERN = re.compile( + r"수신\s*거부|무료거부|" + r"080[-.\s]?\d{3,4}[-.\s]?\d{4}", + re.IGNORECASE, +) + + +# 배열의 열 순서가 학습 및 추론에서 동일해야 하므로 상수로 고정 STACKING_STRUCTURAL_FEATURE_NAMES: tuple[str, ...] = ( "has_url", "has_short_url", @@ -63,6 +76,8 @@ "has_personal_info_request", "has_link_action", "is_long_text", + "has_ad_disclosure", + "has_opt_out", ) @@ -113,6 +128,8 @@ def extract_stacking_structural_features( bool(PERSONAL_INFO_PATTERN.search(text)), bool(LINK_ACTION_PATTERN.search(text)), len(text) > 100, + bool(AD_DISCLOSURE_PATTERN.search(text)), + bool(OPT_OUT_PATTERN.search(text)), ], dtype=np.float64, ) diff --git a/data_science/SMSModel/artifacts/stacking/v3-baseline/metadata.json b/data_science/SMSModel/artifacts/stacking/v3-baseline/metadata.json new file mode 100644 index 0000000..9902097 --- /dev/null +++ b/data_science/SMSModel/artifacts/stacking/v3-baseline/metadata.json @@ -0,0 +1,72 @@ +{ + "artifact_version": "v3", + "created_at": "2026-08-18T06:07:16.622322+00:00", + "dataset": { + "dataset_fingerprint": "46aa236b5c70453bc5b5e91664f4a43d499fffd9a3f103eec9178a30aab85f22", + "holdout_rows": 280, + "total_csv_rows": 3078, + "training_pool_rows": 804 + }, + "dataset_path": "Data/SMSData/phishing_total_dataset_reclassified.csv", + "library_versions": { + "joblib": "1.5.3", + "numpy": "2.4.6", + "pandas": "2.3.3", + "python": "3.13.14", + "scikit_learn": "1.8.0", + "scipy": "1.18.0" + }, + "model": { + "base_models": [ + "naive_bayes", + "logistic_regression", + "linear_svm" + ], + "meta_classifier": "LogisticRegression", + "meta_feature_names": [ + "naive_bayes_score", + "logistic_regression_score", + "linear_svm_score", + "has_url", + "has_short_url", + "has_phone", + "has_account", + "has_card", + "has_amount", + "has_web_tag", + "has_urgency", + "has_transfer_request", + "has_personal_info_request", + "has_link_action", + "is_long_text" + ], + "model_name": "stacking_phishing_classifier", + "oof_splits": 5, + "random_state": 42, + "threshold": 0.30448047609435736 + }, + "model_configuration_sha256": "702b42d87ebbecea9dab08a809a1a38d324a4653a44c7e8e98d095e0431159b4", + "model_sha256": "335ad70ab86b59fc55581e08060f70bc489ec5c1065c13c970ff3b87e717d481", + "schema_version": 2, + "split_manifest": "sms_split_v3.csv", + "split_manifest_sha256": "459c24d86bc73503147d8fbe0844b30ee985a0b2a1e8d4d3981510a124037d2a", + "splits": { + "test": 117, + "train": 565, + "validation": 122 + }, + "training_policy": { + "final_evaluation_split": "test", + "holdout_used_during_training": false, + "random_state": 42, + "test_used_for_tuning": false, + "threshold_selection_split": "validation", + "training_split": "train" + }, + "validation": { + "f2": 0.9061488673139159, + "recall": 0.9655172413793104, + "target_recall": 0.95, + "target_recall_met": true + } +} diff --git a/data_science/SMSModel/artifacts/stacking/v3-baseline/model.joblib b/data_science/SMSModel/artifacts/stacking/v3-baseline/model.joblib new file mode 100644 index 0000000..9fc363d Binary files /dev/null and b/data_science/SMSModel/artifacts/stacking/v3-baseline/model.joblib differ diff --git a/data_science/SMSModel/artifacts/stacking/v3/metadata.json b/data_science/SMSModel/artifacts/stacking/v3/metadata.json index 9902097..3c96be0 100644 --- a/data_science/SMSModel/artifacts/stacking/v3/metadata.json +++ b/data_science/SMSModel/artifacts/stacking/v3/metadata.json @@ -1,6 +1,6 @@ { "artifact_version": "v3", - "created_at": "2026-08-18T06:07:16.622322+00:00", + "created_at": "2026-08-18T08:25:33.056102+00:00", "dataset": { "dataset_fingerprint": "46aa236b5c70453bc5b5e91664f4a43d499fffd9a3f103eec9178a30aab85f22", "holdout_rows": 280, @@ -38,15 +38,17 @@ "has_transfer_request", "has_personal_info_request", "has_link_action", - "is_long_text" + "is_long_text", + "has_ad_disclosure", + "has_opt_out" ], "model_name": "stacking_phishing_classifier", "oof_splits": 5, "random_state": 42, - "threshold": 0.30448047609435736 + "threshold": 0.3038914498086088 }, - "model_configuration_sha256": "702b42d87ebbecea9dab08a809a1a38d324a4653a44c7e8e98d095e0431159b4", - "model_sha256": "335ad70ab86b59fc55581e08060f70bc489ec5c1065c13c970ff3b87e717d481", + "model_configuration_sha256": "8d96d4db57cf1bfc70b73ffa4bd27deaf2c0dbeff6be4c15b2b21d6fd9c5fcdd", + "model_sha256": "72cfe282beaba96e5c7c864ec3a2431e1f9ae41f25a2369a616abcbd6927886f", "schema_version": 2, "split_manifest": "sms_split_v3.csv", "split_manifest_sha256": "459c24d86bc73503147d8fbe0844b30ee985a0b2a1e8d4d3981510a124037d2a", @@ -64,7 +66,7 @@ "training_split": "train" }, "validation": { - "f2": 0.9061488673139159, + "f2": 0.9427609427609428, "recall": 0.9655172413793104, "target_recall": 0.95, "target_recall_met": true diff --git a/data_science/SMSModel/artifacts/stacking/v3/model.joblib b/data_science/SMSModel/artifacts/stacking/v3/model.joblib index 9fc363d..14f3a26 100644 Binary files a/data_science/SMSModel/artifacts/stacking/v3/model.joblib and b/data_science/SMSModel/artifacts/stacking/v3/model.joblib differ diff --git a/data_science/SMSModel/reports/error_analysis_baseline.json b/data_science/SMSModel/reports/error_analysis_baseline.json new file mode 100644 index 0000000..b3a38c0 --- /dev/null +++ b/data_science/SMSModel/reports/error_analysis_baseline.json @@ -0,0 +1,759 @@ +{ + "artifact_threshold": 0.30448047609435736, + "splits": [ + { + "extreme_samples": { + "highest_scoring_normals": [ + { + "probability": 0.6465820939753482, + "text_fingerprint": "1fb552072de451eb1bb5c729af2fa146b85e047dfad59c57cc5a8af5ba866ffe", + "type": "정상금융알림" + }, + { + "probability": 0.5871169412428505, + "text_fingerprint": "7b03ef9bdb9069327e164e255526d8779ae45959641b91eb7e5c0f7b74cd17de", + "type": "정상택배배송안내" + }, + { + "probability": 0.5542466795536436, + "text_fingerprint": "b6b057b5ff1c2ebf44d6b78e163265550a186a0060cadb0e88c96d9195050ccd", + "type": "정상공공기관알림" + }, + { + "probability": 0.5314402941533101, + "text_fingerprint": "a647053db5c41bb55c6e15a84768bffc19517d315d36f5e738d25c67cb58d819", + "type": "정상광고프로모션" + }, + { + "probability": 0.5311101610587133, + "text_fingerprint": "996b10e4d1bdf451c52d001bc98f09750c3dd0765dc17d89e9380ead94a64fb2", + "type": "정상광고프로모션" + }, + { + "probability": 0.5308645962086957, + "text_fingerprint": "805ed16a91a4291aa0621c3864aea2818c628200d2a7bbd5d5b8c6a2d5254c7c", + "type": "정상광고프로모션" + }, + { + "probability": 0.5308496293443415, + "text_fingerprint": "a2a54c20baa1a847779c115c12177f9cbf83747640d965818f69e09c31195f11", + "type": "정상광고프로모션" + }, + { + "probability": 0.5306759862906408, + "text_fingerprint": "374a22505f11d0bfd26dd4f7eddaeab5cfd1978dad6aeacdd99760345d9e7889", + "type": "정상광고프로모션" + }, + { + "probability": 0.5304708595183396, + "text_fingerprint": "c88ae2a464820967ecb667ecf17abab5971c86b3f03a8a03f25aec8542dd9c48", + "type": "정상공공기관알림" + }, + { + "probability": 0.5298819828566479, + "text_fingerprint": "0e615393364bc7f64b3820c0adda8cbb335058ee79028d415183d861e3f84036", + "type": "정상광고프로모션" + }, + { + "probability": 0.5290175239330316, + "text_fingerprint": "f5500e8b3f44592d1162b6fba49096384fda142d0bf0ca48af7ff4a621ad08e5", + "type": "정상광고프로모션" + }, + { + "probability": 0.5285392301996151, + "text_fingerprint": "202ae0604e045296aaf9607396c0d0930b0324236c7b179e35138e5705ca452f", + "type": "정상광고프로모션" + }, + { + "probability": 0.5284579644484564, + "text_fingerprint": "b33f4f2137a05857ba776ee7fd6cb21bfcee0113a70b5c1c539baab0578df19d", + "type": "정상광고프로모션" + }, + { + "probability": 0.5279247981694405, + "text_fingerprint": "9f314b447d1d696796d23e14d27d9313385b8cc8e96dd0ff24e8a24d65351bce", + "type": "정상광고프로모션" + }, + { + "probability": 0.5276646796354805, + "text_fingerprint": "f5d4c2c1554542ff5672baa711b88287c5f0f79ca58864c339451caa2034884a", + "type": "정상광고프로모션" + } + ], + "lowest_scoring_phishing": [ + { + "probability": 0.13968249172707203, + "text_fingerprint": "bac58b099fd068e975a6b06369cf4f17d8b9032f1aa8bc00ef8bf86b4e651a77", + "type": "중고거래사기" + }, + { + "probability": 0.1819429981158037, + "text_fingerprint": "d87d8668d828ff24a01a68de28c96b0c0fa384ceb6fe5819de16b50fe9647d17", + "type": "금융기관사칭" + }, + { + "probability": 0.30448047609435736, + "text_fingerprint": "bf56cdc7392e5da03fa2ac37b7c88f967f78316faa115ce6e7b112b1a61e83f8", + "type": "금융기관사칭" + }, + { + "probability": 0.30548714536884214, + "text_fingerprint": "591d1d3671d7ddba2e5739d9a73359500dd3d8c2b416fc625513db77945d51d6", + "type": "금융기관사칭" + }, + { + "probability": 0.3068110857374601, + "text_fingerprint": "54c7f65b612ca668bc04d3149ff8d829f08ba3f4a8291d38557340422432f197", + "type": "금융기관사칭" + }, + { + "probability": 0.3123667767398097, + "text_fingerprint": "aaddf84eb21b5ec684a769f73db5aaf99c4096826cef79cf04599a5d8d69e74e", + "type": "금융기관사칭" + }, + { + "probability": 0.33221804915805414, + "text_fingerprint": "461cda61772f816922fc24a6e79db8c0d942e3747eba7a4c8356842298ef03f6", + "type": "금융기관사칭" + }, + { + "probability": 0.3348624215700602, + "text_fingerprint": "cf1ef4789f675d47f01afb1e54ac37f6682556f93dc861b0b1f2d8eef327cd29", + "type": "금융기관사칭" + }, + { + "probability": 0.3378441335576765, + "text_fingerprint": "5ee3eac81b36eadf0badc4a7499e89d5dc2b340f76eb7d44836b859514fa2a73", + "type": "금융기관사칭" + }, + { + "probability": 0.3383895712114628, + "text_fingerprint": "ee9ea5e201ddcfbfb677f0a281d4e2dceb2a1bb22274de4a9ea556ab6102a068", + "type": "금융기관사칭" + }, + { + "probability": 0.3384815100440497, + "text_fingerprint": "cc5dae5f1cb4a5365e3b7c5a3adcab875e9bdf01d34368407968ce45073df0e7", + "type": "금융기관사칭" + }, + { + "probability": 0.33960830949667786, + "text_fingerprint": "bb6579844879096b79d10f1b04f46209cf9c1d895627aee57800c468341f7686", + "type": "금융기관사칭" + }, + { + "probability": 0.3409689177781814, + "text_fingerprint": "5e7cbbd5d9e8b24915fb87c360ee45393242fbe8c09c97503937e457927764a4", + "type": "대출사기" + }, + { + "probability": 0.3449281140411607, + "text_fingerprint": "b5e1883331ca8d274bf06474d5078ed3086127e15e9c449f74e09dca06104e58", + "type": "금융기관사칭" + }, + { + "probability": 0.35206103453336896, + "text_fingerprint": "279f61be1f75cf521897e86a237590d1d287b833dcf8364971b41d4f82ea40f7", + "type": "금융기관사칭" + } + ] + }, + "false_positives_at_artifact_threshold": { + "by_type": { + "기타정상": { + "false_positive": 3, + "false_positive_rate": 0.6, + "sample_count": 5 + }, + "일상대화": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 21 + }, + "정상공공기관알림": { + "false_positive": 2, + "false_positive_rate": 0.4, + "sample_count": 5 + }, + "정상광고프로모션": { + "false_positive": 12, + "false_positive_rate": 1.0, + "sample_count": 12 + }, + "정상금융알림": { + "false_positive": 2, + "false_positive_rate": 0.4, + "sample_count": 5 + }, + "정상인증알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 5 + }, + "정상카드결제알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 4 + }, + "정상택배배송안내": { + "false_positive": 2, + "false_positive_rate": 0.4, + "sample_count": 5 + }, + "정상포인트소멸알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 2 + } + }, + "false_positive_count": 21, + "false_positive_rate": 0.328125, + "normal_count": 64, + "threshold": 0.30448047609435736 + }, + "normal_count": 64, + "operating_points": [ + { + "max_recall": 0.6379310344827587, + "missed_phishing": 21, + "target_fpr": 0.02, + "threshold": 0.588132001275363 + }, + { + "max_recall": 0.6896551724137931, + "missed_phishing": 18, + "target_fpr": 0.05, + "threshold": 0.5560003754135029 + }, + { + "max_recall": 0.6896551724137931, + "missed_phishing": 18, + "target_fpr": 0.1, + "threshold": 0.5560003754135029 + }, + { + "max_recall": 0.6896551724137931, + "missed_phishing": 18, + "target_fpr": 0.15, + "threshold": 0.5560003754135029 + }, + { + "max_recall": 0.6896551724137931, + "missed_phishing": 18, + "target_fpr": 0.2, + "threshold": 0.5560003754135029 + } + ], + "phishing_count": 58, + "pr_auc": 0.9011808316780805, + "roc_auc": 0.8895474137931034, + "sample_count": 122, + "split": "validation" + }, + { + "extreme_samples": { + "highest_scoring_normals": [ + { + "probability": 0.57546496124326, + "text_fingerprint": "5580c99edd3218684721ed377d478d39496755e1956427c69ec48ccedee48db8", + "type": "정상공공기관알림" + }, + { + "probability": 0.5118051383256766, + "text_fingerprint": "4179f54dfe26365994664664a8621684d1b47818b3588d81bd890449acf218ee", + "type": "기타정상" + }, + { + "probability": 0.39602025178758327, + "text_fingerprint": "dd9c13a2c2f2747da2a12764030c3b1df5dbeb94dde8116dd4b4cdf9e93df172", + "type": "기타정상" + }, + { + "probability": 0.392016482490789, + "text_fingerprint": "ccce2b7f00a16f8446b78440ca74d229f7f29faf08ae1b8b34ddebc43bee98fb", + "type": "정상금융알림" + }, + { + "probability": 0.36445309549301425, + "text_fingerprint": "dc3b31cf3ae71d5c7accc6ac74255783ae77a04394219d290d1b53f7d791213c", + "type": "정상택배배송안내" + }, + { + "probability": 0.35974095270139544, + "text_fingerprint": "a5bd16e9e8c638a07feac44cc91021ad0b5b5a56523b4f7a52050065625e0907", + "type": "정상공공기관알림" + }, + { + "probability": 0.339042162257523, + "text_fingerprint": "e14cafbff49aa7a3e2397a2ede7aeef4ff0003e2a3a9c144b1139a0597211374", + "type": "정상공공기관알림" + }, + { + "probability": 0.32800160715274823, + "text_fingerprint": "e142cdd243127df2be02681d70ab63c0727d0f5ea21b8fe01e32aac6f0a5f7d6", + "type": "정상광고프로모션" + }, + { + "probability": 0.31254025841635513, + "text_fingerprint": "c93b8bcf3754cd8d9921df04f274c9bbadd47527e802113dd4751ddbd96ad7bd", + "type": "정상택배배송안내" + }, + { + "probability": 0.3062834186654916, + "text_fingerprint": "95adda6bb56eadb664fafd78110e4ad88f22880471679ffb558176b6da1c3153", + "type": "정상광고프로모션" + }, + { + "probability": 0.3057309848309791, + "text_fingerprint": "4500e37aa1d42963021b06531c284b87f20d0c6f558084f04a52a9a34fabb42c", + "type": "정상광고프로모션" + }, + { + "probability": 0.3050189380453232, + "text_fingerprint": "3ffe7351bbaae187b623b7d3f36883ce33291d3bd21776a744e1ce49899884be", + "type": "정상광고프로모션" + }, + { + "probability": 0.3049267576525945, + "text_fingerprint": "7e3aabb364ce57866b881da6ef7dd6b953268e05267a323c72a50799df39731f", + "type": "정상광고프로모션" + }, + { + "probability": 0.30314721653100524, + "text_fingerprint": "f9cf49989bd91792719aa506fa701db83db29c35863fcb7e555ca854d14ba637", + "type": "정상광고프로모션" + }, + { + "probability": 0.30085299094581236, + "text_fingerprint": "1b538383bd482eac3211546f3eb77fe3340f71e1a507f1727bbcd7c5554e4ba7", + "type": "정상광고프로모션" + } + ], + "lowest_scoring_phishing": [ + { + "probability": 0.08445220879959636, + "text_fingerprint": "69f97bb6c8973d864ca30434d19ec6e7fa9ff2900b017c07d47f4af10b43fff3", + "type": "정부공공기관사칭" + }, + { + "probability": 0.24652573771982822, + "text_fingerprint": "f486cca20e58cf0e8ab98706d819579adab277a57e29852b607f948e94e6e715", + "type": "지인가족사칭" + }, + { + "probability": 0.2526114399271795, + "text_fingerprint": "10473b8eb05ba7f1a3c9e53b8f198a1eabf368220622ab34f18a144595a3dc28", + "type": "악성링크앱설치유도" + }, + { + "probability": 0.288045073080074, + "text_fingerprint": "fd2d19d83512b7149a8be03805c12950e64fe7db3708134956cac385580a37ab", + "type": "투자리딩방사기" + }, + { + "probability": 0.3646701012577986, + "text_fingerprint": "a0d2d59f63390887bd0bd9e2dc90c4a0af67bec8b87579c1982fa335d1136a04", + "type": "채용부업사기" + }, + { + "probability": 0.43204881552684643, + "text_fingerprint": "b9d87b03878c6d21e555baac6698d2371eecb7bbc3e4a0a0d6ca97640535e6cb", + "type": "금융기관사칭" + }, + { + "probability": 0.46021371425298907, + "text_fingerprint": "102ecec1e3c66b3d3b2fee43a9b766dfbeea6807a2862be01d6e115b97ab909e", + "type": "기타피싱" + }, + { + "probability": 0.4869751673037397, + "text_fingerprint": "77b81d18dc5de0e4539eae4029aed9b4e8c1dd6ba45dd38469449f488a4ce4d2", + "type": "계정정지본인인증유도" + }, + { + "probability": 0.5444918858450771, + "text_fingerprint": "e1dd61e8f30da5722021c4f3da82bdadfcc56271431f108bb0f59a932aa15430", + "type": "금융기관사칭" + }, + { + "probability": 0.5661994296241277, + "text_fingerprint": "6954d76bffa4df64d32eece72440e5da18f1f6546b2f98ec65b344bbdfac4056", + "type": "금융기관사칭" + }, + { + "probability": 0.5943615332837933, + "text_fingerprint": "52a0a2312e8f0647186a76def39b78b184e31c21bce310a61f154aab8378989e", + "type": "금융기관사칭" + }, + { + "probability": 0.6122784520937824, + "text_fingerprint": "0664ce67156ce04829df8ff3413da46c3ecdf48674594506efd5cf4ea692b4f5", + "type": "금융기관사칭" + }, + { + "probability": 0.6146732124506805, + "text_fingerprint": "d5c1f60e37f3b985c494e304e4421d6134a34f5b189745919d77a8949ddff57f", + "type": "금융기관사칭" + }, + { + "probability": 0.621248088094708, + "text_fingerprint": "6963063fc3c9796c06e46059dbbcd00d021b820b9d2374a82313f617fa65820f", + "type": "중고거래사기" + }, + { + "probability": 0.6370556045876463, + "text_fingerprint": "1b4632392cc594cbd7f73cedfdb7cfe61384a76a8937ec34c9709142d490f8e9", + "type": "결제환불사칭" + } + ] + }, + "false_positives_at_artifact_threshold": { + "by_type": { + "기타정상": { + "false_positive": 2, + "false_positive_rate": 0.4, + "sample_count": 5 + }, + "일상대화": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 21 + }, + "정상공공기관알림": { + "false_positive": 3, + "false_positive_rate": 0.75, + "sample_count": 4 + }, + "정상광고프로모션": { + "false_positive": 5, + "false_positive_rate": 0.625, + "sample_count": 8 + }, + "정상금융알림": { + "false_positive": 1, + "false_positive_rate": 0.25, + "sample_count": 4 + }, + "정상인증알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 4 + }, + "정상카드결제알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 4 + }, + "정상택배배송안내": { + "false_positive": 2, + "false_positive_rate": 0.3333333333333333, + "sample_count": 6 + }, + "정상포인트소멸알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 2 + } + }, + "false_positive_count": 13, + "false_positive_rate": 0.22413793103448276, + "normal_count": 58, + "threshold": 0.30448047609435736 + }, + "normal_count": 58, + "operating_points": [ + { + "max_recall": 0.864406779661017, + "missed_phishing": 8, + "target_fpr": 0.02, + "threshold": 0.5444918858450771 + }, + { + "max_recall": 0.9152542372881356, + "missed_phishing": 5, + "target_fpr": 0.05, + "threshold": 0.43204881552684643 + }, + { + "max_recall": 0.9322033898305084, + "missed_phishing": 4, + "target_fpr": 0.1, + "threshold": 0.3646701012577986 + }, + { + "max_recall": 0.9322033898305084, + "missed_phishing": 4, + "target_fpr": 0.15, + "threshold": 0.3646701012577986 + }, + { + "max_recall": 0.9322033898305084, + "missed_phishing": 4, + "target_fpr": 0.2, + "threshold": 0.3646701012577986 + } + ], + "phishing_count": 59, + "pr_auc": 0.9755033958553341, + "roc_auc": 0.9643483343074226, + "sample_count": 117, + "split": "test" + }, + { + "extreme_samples": { + "highest_scoring_normals": [ + { + "probability": 0.8688748975200682, + "text_fingerprint": "15e3408fe377c99e7b7d3c1ebd0f20551abe2c63b72487ccb2e734b009a2e64c", + "type": "정상광고프로모션" + }, + { + "probability": 0.85831521302923, + "text_fingerprint": "006cd7018b69018d6fcde02725148d6ccb30c7a6623a5f6d7cc07da24cff3750", + "type": "정상광고프로모션" + }, + { + "probability": 0.8330039399620017, + "text_fingerprint": "0a97cb36007c0aa95a9c92280832d3944e63702c74e75f4394b90ff0f4ee0774", + "type": "기타정상" + }, + { + "probability": 0.7513231350986095, + "text_fingerprint": "2836706edb74f93eaf81cda99e04d8c593f07e6f440a8429c89a8ece1558a330", + "type": "정상광고프로모션" + }, + { + "probability": 0.6945511737437579, + "text_fingerprint": "10230255369903082a88920fac2895b0714dec5e6ff7b2a729be4d23ace84d51", + "type": "정상광고프로모션" + }, + { + "probability": 0.48417527953869743, + "text_fingerprint": "1fed23922041561e7e1f8a07f25a8c653e02da9f17d99b2a00fb9adac9e4c55b", + "type": "정상광고프로모션" + }, + { + "probability": 0.4807654050749924, + "text_fingerprint": "13a9d1849e20647058ff4c6f7da202520e3cd230aad38a24629e20471e5590c9", + "type": "기타정상" + }, + { + "probability": 0.4658646614147581, + "text_fingerprint": "534c8fd679825d3fe1f006f271adb420465ef772ba31a9bcb5d8d10f0c8a2db7", + "type": "정상광고프로모션" + }, + { + "probability": 0.4598889304580791, + "text_fingerprint": "73f7dc0cbe8e31fee604c0f4ddd7983bcd9b093c9fcd7390a9260e4a8b6c7008", + "type": "정상광고프로모션" + }, + { + "probability": 0.342268030783543, + "text_fingerprint": "4142fcb037bf6abb39e3c8d499beba8b3f08276d63f3f041911c1ebd7a574222", + "type": "정상광고프로모션" + }, + { + "probability": 0.3283809571309929, + "text_fingerprint": "900b6706739a4a9c67027eb00c34c64b8a073b62bc95aa3f667dab2e19d45560", + "type": "정상카드결제알림" + }, + { + "probability": 0.2854809798314344, + "text_fingerprint": "1717bc55259a1f8214b2c3b3af4fd234e59bd24bf774b35d3bfdcfb62c70d96f", + "type": "기타정상" + }, + { + "probability": 0.2458761335515605, + "text_fingerprint": "09555dd484584de2b36b5bb7aad6ce432ad88655f7a49def3059f9cd896e5ceb", + "type": "기타정상" + }, + { + "probability": 0.23200854367933993, + "text_fingerprint": "2156329f036113c89e3ec1e1060eb3c09a2303a0a4a2f058d73ed7f634d634c2", + "type": "정상금융알림" + }, + { + "probability": 0.21702337838858626, + "text_fingerprint": "5719ad163e2e66c931c6c8543e5bf4a9a93539ccb9a6f491640c5f22b3ad2cd4", + "type": "정상택배배송안내" + } + ], + "lowest_scoring_phishing": [ + { + "probability": 0.08569011169253814, + "text_fingerprint": "829fea59062a71258522b9c5cc465d68b271418a21b667825e9e41e699928603", + "type": "지인가족사칭" + }, + { + "probability": 0.14506187128794934, + "text_fingerprint": "a81efa689e5bab91577bbfd3c0ae3e32a941d3c8c7b18566cc15e6abf4479070", + "type": "투자리딩방사기" + }, + { + "probability": 0.14816468150484136, + "text_fingerprint": "4f497e918721762642d62776843e758fe66b7c594b937b4c16fee9697f49e56b", + "type": "금융기관사칭" + }, + { + "probability": 0.16506316545559463, + "text_fingerprint": "2717c495b40216b748632e4da0eb0a30dc073b9c8a77582aaab21ac0b7ea1539", + "type": "채용부업사기" + }, + { + "probability": 0.21270005793815694, + "text_fingerprint": "3a6048f2cc45147d555c7416de1fbf811a438bb9ddb9b057ab2d38fead39b9c0", + "type": "투자리딩방사기" + }, + { + "probability": 0.2164555915781086, + "text_fingerprint": "6042b81b11cfa5a8c9db730e483d9ca119afda3caa0a9e2831fa3ca066689d92", + "type": "지인가족사칭" + }, + { + "probability": 0.26703949201083244, + "text_fingerprint": "9a17e496aea032ec4edc7355bfa89673f14a14bd017bec19bdc0c316cacd41b4", + "type": "투자리딩방사기" + }, + { + "probability": 0.28371878559678837, + "text_fingerprint": "61ccd047a6d5f78a71f945165a65d40f04f8c92424d699d6319c5af94682c7fb", + "type": "택배배송사칭" + }, + { + "probability": 0.3023263624633487, + "text_fingerprint": "2fbc6b4e11113b0d39ced22dcfda822d3b64760260a479766403efa1ef49e591", + "type": "지인가족사칭" + }, + { + "probability": 0.32874032651610036, + "text_fingerprint": "e748c7274d6ce8ee555f9f8cd5e86a67927ef282c831ed5ace423f7e7a7a4cc1", + "type": "지인가족사칭" + }, + { + "probability": 0.3405678881779246, + "text_fingerprint": "d5b9a5c7d227f2f6a0dbe6d268581389963019893ce9564553a310334234586f", + "type": "금융기관사칭" + }, + { + "probability": 0.3526444101623605, + "text_fingerprint": "5cac45fd0afdc096318548b3a06032cd938689d86025d9e81e2e3fc76e14b33f", + "type": "금융기관사칭" + }, + { + "probability": 0.37962082716919276, + "text_fingerprint": "cf3a5f447f417e56a6acd365134a427a10ac98f7a9b2566b55ef7ff3a82b8cb4", + "type": "금융기관사칭" + }, + { + "probability": 0.38648153596489926, + "text_fingerprint": "46335873d6264088f204441052209785487588dec4113a6a4bcbfb7e62fe15ff", + "type": "금융기관사칭" + }, + { + "probability": 0.41706558760274065, + "text_fingerprint": "db6fa1e8200183d485a1386d0d097415a832912f89d9c6c9079a9c91020e4955", + "type": "지인가족사칭" + } + ] + }, + "false_positives_at_artifact_threshold": { + "by_type": { + "기타정상": { + "false_positive": 2, + "false_positive_rate": 0.15384615384615385, + "sample_count": 13 + }, + "일상대화": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 34 + }, + "정상광고프로모션": { + "false_positive": 8, + "false_positive_rate": 1.0, + "sample_count": 8 + }, + "정상금융알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 6 + }, + "정상카드결제알림": { + "false_positive": 1, + "false_positive_rate": 0.25, + "sample_count": 4 + }, + "정상택배배송안내": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 6 + }, + "정상포인트소멸알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 3 + } + }, + "false_positive_count": 11, + "false_positive_rate": 0.14864864864864866, + "normal_count": 74, + "threshold": 0.30448047609435736 + }, + "normal_count": 74, + "operating_points": [ + { + "max_recall": 0.5060240963855421, + "missed_phishing": 41, + "target_fpr": 0.02, + "threshold": 0.8728328730581757 + }, + { + "max_recall": 0.5783132530120482, + "missed_phishing": 35, + "target_fpr": 0.05, + "threshold": 0.7872506080456074 + }, + { + "max_recall": 0.7951807228915663, + "missed_phishing": 17, + "target_fpr": 0.1, + "threshold": 0.47349007617280453 + }, + { + "max_recall": 0.9036144578313253, + "missed_phishing": 8, + "target_fpr": 0.15, + "threshold": 0.3023263624633487 + }, + { + "max_recall": 0.927710843373494, + "missed_phishing": 6, + "target_fpr": 0.2, + "threshold": 0.26703949201083244 + } + ], + "phishing_count": 83, + "pr_auc": 0.9446839484837867, + "roc_auc": 0.9356887007489417, + "sample_count": 157, + "split": "real_holdout" + } + ], + "structural_feature_names": [ + "naive_bayes_score", + "logistic_regression_score", + "linear_svm_score", + "has_url", + "has_short_url", + "has_phone", + "has_account", + "has_card", + "has_amount", + "has_web_tag", + "has_urgency", + "has_transfer_request", + "has_personal_info_request", + "has_link_action", + "is_long_text" + ] +} diff --git a/data_science/SMSModel/reports/error_analysis_with_ad_features.json b/data_science/SMSModel/reports/error_analysis_with_ad_features.json new file mode 100644 index 0000000..667529c --- /dev/null +++ b/data_science/SMSModel/reports/error_analysis_with_ad_features.json @@ -0,0 +1,761 @@ +{ + "artifact_threshold": 0.3038914498086088, + "splits": [ + { + "extreme_samples": { + "highest_scoring_normals": [ + { + "probability": 0.6386041598664511, + "text_fingerprint": "1fb552072de451eb1bb5c729af2fa146b85e047dfad59c57cc5a8af5ba866ffe", + "type": "정상금융알림" + }, + { + "probability": 0.59311774739082, + "text_fingerprint": "7b03ef9bdb9069327e164e255526d8779ae45959641b91eb7e5c0f7b74cd17de", + "type": "정상택배배송안내" + }, + { + "probability": 0.5350483469296247, + "text_fingerprint": "c88ae2a464820967ecb667ecf17abab5971c86b3f03a8a03f25aec8542dd9c48", + "type": "정상공공기관알림" + }, + { + "probability": 0.5306083370006258, + "text_fingerprint": "b6b057b5ff1c2ebf44d6b78e163265550a186a0060cadb0e88c96d9195050ccd", + "type": "정상공공기관알림" + }, + { + "probability": 0.5290042451271526, + "text_fingerprint": "c2c90aec6b8691fa77c0cd637c92c75fc4023ce987e471fbded32df5a8e1dcd4", + "type": "기타정상" + }, + { + "probability": 0.5202521676362787, + "text_fingerprint": "6b818d1bcc889de492158cbb27cc6fead5196d4d92e194550c0617b9ea94c9ce", + "type": "기타정상" + }, + { + "probability": 0.40453196024864646, + "text_fingerprint": "63fac6a7e948aa2daef3f106f0315a48e2361f6cd07047ce725966ccd42075cc", + "type": "기타정상" + }, + { + "probability": 0.3827525463089185, + "text_fingerprint": "f5632aa15a179afb0a9bda37a4f4872d0507a8d1cb5b9b194313f5b0e4164453", + "type": "정상택배배송안내" + }, + { + "probability": 0.32504561143179994, + "text_fingerprint": "f020a95f3f754372feb3c83fba5afa292552cb9435648b39e9429ed5caaf5e55", + "type": "정상금융알림" + }, + { + "probability": 0.29562744929134327, + "text_fingerprint": "092ebe92b646037397c774d3040b9a6114947b9c18ff8e3a10e7e02f58d41871", + "type": "정상인증알림" + }, + { + "probability": 0.2542686098396349, + "text_fingerprint": "5a40c3f63bec6e17241934bb1eaa1f04034fab09c6fc824f10926c01fc71e43d", + "type": "일상대화" + }, + { + "probability": 0.22475953809925706, + "text_fingerprint": "6f1c934314a7432ba7fe74f4a7986ab208996cbc30394617e90203191e49535d", + "type": "정상금융알림" + }, + { + "probability": 0.21528281004080768, + "text_fingerprint": "928cb457932441a4bccbf68334635089fa94fcad1c56f0f0264372960fc2d2ae", + "type": "기타정상" + }, + { + "probability": 0.2093440727870738, + "text_fingerprint": "a647053db5c41bb55c6e15a84768bffc19517d315d36f5e738d25c67cb58d819", + "type": "정상광고프로모션" + }, + { + "probability": 0.2091198639333299, + "text_fingerprint": "996b10e4d1bdf451c52d001bc98f09750c3dd0765dc17d89e9380ead94a64fb2", + "type": "정상광고프로모션" + } + ], + "lowest_scoring_phishing": [ + { + "probability": 0.1349245720515813, + "text_fingerprint": "bac58b099fd068e975a6b06369cf4f17d8b9032f1aa8bc00ef8bf86b4e651a77", + "type": "중고거래사기" + }, + { + "probability": 0.19766478071778712, + "text_fingerprint": "d87d8668d828ff24a01a68de28c96b0c0fa384ceb6fe5819de16b50fe9647d17", + "type": "금융기관사칭" + }, + { + "probability": 0.3038914498086088, + "text_fingerprint": "bf56cdc7392e5da03fa2ac37b7c88f967f78316faa115ce6e7b112b1a61e83f8", + "type": "금융기관사칭" + }, + { + "probability": 0.30491934227769524, + "text_fingerprint": "591d1d3671d7ddba2e5739d9a73359500dd3d8c2b416fc625513db77945d51d6", + "type": "금융기관사칭" + }, + { + "probability": 0.3062712673198561, + "text_fingerprint": "54c7f65b612ca668bc04d3149ff8d829f08ba3f4a8291d38557340422432f197", + "type": "금융기관사칭" + }, + { + "probability": 0.31191686307914046, + "text_fingerprint": "aaddf84eb21b5ec684a769f73db5aaf99c4096826cef79cf04599a5d8d69e74e", + "type": "금융기관사칭" + }, + { + "probability": 0.32773767776669765, + "text_fingerprint": "5e7cbbd5d9e8b24915fb87c360ee45393242fbe8c09c97503937e457927764a4", + "type": "대출사기" + }, + { + "probability": 0.33209429214521374, + "text_fingerprint": "461cda61772f816922fc24a6e79db8c0d942e3747eba7a4c8356842298ef03f6", + "type": "금융기관사칭" + }, + { + "probability": 0.3348014108100571, + "text_fingerprint": "cf1ef4789f675d47f01afb1e54ac37f6682556f93dc861b0b1f2d8eef327cd29", + "type": "금융기관사칭" + }, + { + "probability": 0.33784517954749826, + "text_fingerprint": "5ee3eac81b36eadf0badc4a7499e89d5dc2b340f76eb7d44836b859514fa2a73", + "type": "금융기관사칭" + }, + { + "probability": 0.33840248886270746, + "text_fingerprint": "ee9ea5e201ddcfbfb677f0a281d4e2dceb2a1bb22274de4a9ea556ab6102a068", + "type": "금융기관사칭" + }, + { + "probability": 0.33849520932926835, + "text_fingerprint": "cc5dae5f1cb4a5365e3b7c5a3adcab875e9bdf01d34368407968ce45073df0e7", + "type": "금융기관사칭" + }, + { + "probability": 0.33964694617196417, + "text_fingerprint": "bb6579844879096b79d10f1b04f46209cf9c1d895627aee57800c468341f7686", + "type": "금융기관사칭" + }, + { + "probability": 0.34509424918981074, + "text_fingerprint": "b5e1883331ca8d274bf06474d5078ed3086127e15e9c449f74e09dca06104e58", + "type": "금융기관사칭" + }, + { + "probability": 0.3523795472479009, + "text_fingerprint": "279f61be1f75cf521897e86a237590d1d287b833dcf8364971b41d4f82ea40f7", + "type": "금융기관사칭" + } + ] + }, + "false_positives_at_artifact_threshold": { + "by_type": { + "기타정상": { + "false_positive": 3, + "false_positive_rate": 0.6, + "sample_count": 5 + }, + "일상대화": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 21 + }, + "정상공공기관알림": { + "false_positive": 2, + "false_positive_rate": 0.4, + "sample_count": 5 + }, + "정상광고프로모션": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 12 + }, + "정상금융알림": { + "false_positive": 2, + "false_positive_rate": 0.4, + "sample_count": 5 + }, + "정상인증알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 5 + }, + "정상카드결제알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 4 + }, + "정상택배배송안내": { + "false_positive": 2, + "false_positive_rate": 0.4, + "sample_count": 5 + }, + "정상포인트소멸알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 2 + } + }, + "false_positive_count": 9, + "false_positive_rate": 0.140625, + "normal_count": 64, + "threshold": 0.3038914498086088 + }, + "normal_count": 64, + "operating_points": [ + { + "max_recall": 0.6551724137931034, + "missed_phishing": 20, + "target_fpr": 0.02, + "threshold": 0.6155609963151684 + }, + { + "max_recall": 0.6896551724137931, + "missed_phishing": 18, + "target_fpr": 0.05, + "threshold": 0.5619157863250824 + }, + { + "max_recall": 0.7068965517241379, + "missed_phishing": 17, + "target_fpr": 0.1, + "threshold": 0.45322848989655073 + }, + { + "max_recall": 0.9655172413793104, + "missed_phishing": 2, + "target_fpr": 0.15, + "threshold": 0.3038914498086088 + }, + { + "max_recall": 0.9655172413793104, + "missed_phishing": 2, + "target_fpr": 0.2, + "threshold": 0.3038914498086088 + } + ], + "phishing_count": 58, + "pr_auc": 0.9455148761039095, + "roc_auc": 0.9447737068965517, + "sample_count": 122, + "split": "validation" + }, + { + "extreme_samples": { + "highest_scoring_normals": [ + { + "probability": 0.7424612952832208, + "text_fingerprint": "5580c99edd3218684721ed377d478d39496755e1956427c69ec48ccedee48db8", + "type": "정상공공기관알림" + }, + { + "probability": 0.4820551670971015, + "text_fingerprint": "4179f54dfe26365994664664a8621684d1b47818b3588d81bd890449acf218ee", + "type": "기타정상" + }, + { + "probability": 0.4490144113056518, + "text_fingerprint": "dd9c13a2c2f2747da2a12764030c3b1df5dbeb94dde8116dd4b4cdf9e93df172", + "type": "기타정상" + }, + { + "probability": 0.36670959731489977, + "text_fingerprint": "dc3b31cf3ae71d5c7accc6ac74255783ae77a04394219d290d1b53f7d791213c", + "type": "정상택배배송안내" + }, + { + "probability": 0.36545095613322554, + "text_fingerprint": "ccce2b7f00a16f8446b78440ca74d229f7f29faf08ae1b8b34ddebc43bee98fb", + "type": "정상금융알림" + }, + { + "probability": 0.34688580800283103, + "text_fingerprint": "a5bd16e9e8c638a07feac44cc91021ad0b5b5a56523b4f7a52050065625e0907", + "type": "정상공공기관알림" + }, + { + "probability": 0.33926437166371776, + "text_fingerprint": "e14cafbff49aa7a3e2397a2ede7aeef4ff0003e2a3a9c144b1139a0597211374", + "type": "정상공공기관알림" + }, + { + "probability": 0.3308782090776632, + "text_fingerprint": "95adda6bb56eadb664fafd78110e4ad88f22880471679ffb558176b6da1c3153", + "type": "정상광고프로모션" + }, + { + "probability": 0.3302897779794274, + "text_fingerprint": "4500e37aa1d42963021b06531c284b87f20d0c6f558084f04a52a9a34fabb42c", + "type": "정상광고프로모션" + }, + { + "probability": 0.3295312394002752, + "text_fingerprint": "3ffe7351bbaae187b623b7d3f36883ce33291d3bd21776a744e1ce49899884be", + "type": "정상광고프로모션" + }, + { + "probability": 0.3294330325291557, + "text_fingerprint": "7e3aabb364ce57866b881da6ef7dd6b953268e05267a323c72a50799df39731f", + "type": "정상광고프로모션" + }, + { + "probability": 0.32753680378593125, + "text_fingerprint": "f9cf49989bd91792719aa506fa701db83db29c35863fcb7e555ca854d14ba637", + "type": "정상광고프로모션" + }, + { + "probability": 0.32509117249953834, + "text_fingerprint": "1b538383bd482eac3211546f3eb77fe3340f71e1a507f1727bbcd7c5554e4ba7", + "type": "정상광고프로모션" + }, + { + "probability": 0.32408491951379226, + "text_fingerprint": "e142cdd243127df2be02681d70ab63c0727d0f5ea21b8fe01e32aac6f0a5f7d6", + "type": "정상광고프로모션" + }, + { + "probability": 0.3236968173082904, + "text_fingerprint": "b7d667bc51f5c9fe97cb257c1e513e5f019c61be26f6771dfe8843883a5bf29d", + "type": "정상광고프로모션" + } + ], + "lowest_scoring_phishing": [ + { + "probability": 0.09068993664403237, + "text_fingerprint": "69f97bb6c8973d864ca30434d19ec6e7fa9ff2900b017c07d47f4af10b43fff3", + "type": "정부공공기관사칭" + }, + { + "probability": 0.24526907371487472, + "text_fingerprint": "f486cca20e58cf0e8ab98706d819579adab277a57e29852b607f948e94e6e715", + "type": "지인가족사칭" + }, + { + "probability": 0.2508907430495912, + "text_fingerprint": "10473b8eb05ba7f1a3c9e53b8f198a1eabf368220622ab34f18a144595a3dc28", + "type": "악성링크앱설치유도" + }, + { + "probability": 0.27477214800756533, + "text_fingerprint": "fd2d19d83512b7149a8be03805c12950e64fe7db3708134956cac385580a37ab", + "type": "투자리딩방사기" + }, + { + "probability": 0.3093093507801836, + "text_fingerprint": "102ecec1e3c66b3d3b2fee43a9b766dfbeea6807a2862be01d6e115b97ab909e", + "type": "기타피싱" + }, + { + "probability": 0.3654948939741732, + "text_fingerprint": "a0d2d59f63390887bd0bd9e2dc90c4a0af67bec8b87579c1982fa335d1136a04", + "type": "채용부업사기" + }, + { + "probability": 0.483163847632848, + "text_fingerprint": "b9d87b03878c6d21e555baac6698d2371eecb7bbc3e4a0a0d6ca97640535e6cb", + "type": "금융기관사칭" + }, + { + "probability": 0.49101946601555063, + "text_fingerprint": "77b81d18dc5de0e4539eae4029aed9b4e8c1dd6ba45dd38469449f488a4ce4d2", + "type": "계정정지본인인증유도" + }, + { + "probability": 0.6159574574298013, + "text_fingerprint": "1b4632392cc594cbd7f73cedfdb7cfe61384a76a8937ec34c9709142d490f8e9", + "type": "결제환불사칭" + }, + { + "probability": 0.622264920109958, + "text_fingerprint": "6954d76bffa4df64d32eece72440e5da18f1f6546b2f98ec65b344bbdfac4056", + "type": "금융기관사칭" + }, + { + "probability": 0.6289088831075103, + "text_fingerprint": "e1dd61e8f30da5722021c4f3da82bdadfcc56271431f108bb0f59a932aa15430", + "type": "금융기관사칭" + }, + { + "probability": 0.6319133219706925, + "text_fingerprint": "6963063fc3c9796c06e46059dbbcd00d021b820b9d2374a82313f617fa65820f", + "type": "중고거래사기" + }, + { + "probability": 0.6759714029544074, + "text_fingerprint": "52a0a2312e8f0647186a76def39b78b184e31c21bce310a61f154aab8378989e", + "type": "금융기관사칭" + }, + { + "probability": 0.6924418549792178, + "text_fingerprint": "0664ce67156ce04829df8ff3413da46c3ecdf48674594506efd5cf4ea692b4f5", + "type": "금융기관사칭" + }, + { + "probability": 0.6946109122760136, + "text_fingerprint": "d5c1f60e37f3b985c494e304e4421d6134a34f5b189745919d77a8949ddff57f", + "type": "금융기관사칭" + } + ] + }, + "false_positives_at_artifact_threshold": { + "by_type": { + "기타정상": { + "false_positive": 2, + "false_positive_rate": 0.4, + "sample_count": 5 + }, + "일상대화": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 21 + }, + "정상공공기관알림": { + "false_positive": 3, + "false_positive_rate": 0.75, + "sample_count": 4 + }, + "정상광고프로모션": { + "false_positive": 8, + "false_positive_rate": 1.0, + "sample_count": 8 + }, + "정상금융알림": { + "false_positive": 1, + "false_positive_rate": 0.25, + "sample_count": 4 + }, + "정상인증알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 4 + }, + "정상카드결제알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 4 + }, + "정상택배배송안내": { + "false_positive": 2, + "false_positive_rate": 0.3333333333333333, + "sample_count": 6 + }, + "정상포인트소멸알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 2 + } + }, + "false_positive_count": 16, + "false_positive_rate": 0.27586206896551724, + "normal_count": 58, + "threshold": 0.3038914498086088 + }, + "normal_count": 58, + "operating_points": [ + { + "max_recall": 0.8983050847457628, + "missed_phishing": 6, + "target_fpr": 0.02, + "threshold": 0.483163847632848 + }, + { + "max_recall": 0.8983050847457628, + "missed_phishing": 6, + "target_fpr": 0.05, + "threshold": 0.483163847632848 + }, + { + "max_recall": 0.9152542372881356, + "missed_phishing": 5, + "target_fpr": 0.1, + "threshold": 0.3654948939741732 + }, + { + "max_recall": 0.9152542372881356, + "missed_phishing": 5, + "target_fpr": 0.15, + "threshold": 0.3654948939741732 + }, + { + "max_recall": 0.9152542372881356, + "missed_phishing": 5, + "target_fpr": 0.2, + "threshold": 0.3654948939741732 + } + ], + "phishing_count": 59, + "pr_auc": 0.9686415306672431, + "roc_auc": 0.957042665108124, + "sample_count": 117, + "split": "test" + }, + { + "extreme_samples": { + "highest_scoring_normals": [ + { + "probability": 0.8496571018813256, + "text_fingerprint": "0a97cb36007c0aa95a9c92280832d3944e63702c74e75f4394b90ff0f4ee0774", + "type": "기타정상" + }, + { + "probability": 0.7825199908851669, + "text_fingerprint": "15e3408fe377c99e7b7d3c1ebd0f20551abe2c63b72487ccb2e734b009a2e64c", + "type": "정상광고프로모션" + }, + { + "probability": 0.7565591033251493, + "text_fingerprint": "006cd7018b69018d6fcde02725148d6ccb30c7a6623a5f6d7cc07da24cff3750", + "type": "정상광고프로모션" + }, + { + "probability": 0.6414861490042555, + "text_fingerprint": "73f7dc0cbe8e31fee604c0f4ddd7983bcd9b093c9fcd7390a9260e4a8b6c7008", + "type": "정상광고프로모션" + }, + { + "probability": 0.5279710299385563, + "text_fingerprint": "13a9d1849e20647058ff4c6f7da202520e3cd230aad38a24629e20471e5590c9", + "type": "기타정상" + }, + { + "probability": 0.47914129763716895, + "text_fingerprint": "10230255369903082a88920fac2895b0714dec5e6ff7b2a729be4d23ace84d51", + "type": "정상광고프로모션" + }, + { + "probability": 0.43738976517771105, + "text_fingerprint": "2836706edb74f93eaf81cda99e04d8c593f07e6f440a8429c89a8ece1558a330", + "type": "정상광고프로모션" + }, + { + "probability": 0.42546763280125, + "text_fingerprint": "534c8fd679825d3fe1f006f271adb420465ef772ba31a9bcb5d8d10f0c8a2db7", + "type": "정상광고프로모션" + }, + { + "probability": 0.35887049837195717, + "text_fingerprint": "900b6706739a4a9c67027eb00c34c64b8a073b62bc95aa3f667dab2e19d45560", + "type": "정상카드결제알림" + }, + { + "probability": 0.2840637191849847, + "text_fingerprint": "1717bc55259a1f8214b2c3b3af4fd234e59bd24bf774b35d3bfdcfb62c70d96f", + "type": "기타정상" + }, + { + "probability": 0.2779881742333603, + "text_fingerprint": "09555dd484584de2b36b5bb7aad6ce432ad88655f7a49def3059f9cd896e5ceb", + "type": "기타정상" + }, + { + "probability": 0.27200087309737514, + "text_fingerprint": "1fed23922041561e7e1f8a07f25a8c653e02da9f17d99b2a00fb9adac9e4c55b", + "type": "정상광고프로모션" + }, + { + "probability": 0.2650934093653113, + "text_fingerprint": "2156329f036113c89e3ec1e1060eb3c09a2303a0a4a2f058d73ed7f634d634c2", + "type": "정상금융알림" + }, + { + "probability": 0.24178026170119601, + "text_fingerprint": "01fe49492e887f979035a5d103e9e2f95b7e971f5eb072ae0e5393e3fdcea16f", + "type": "기타정상" + }, + { + "probability": 0.21965644123332467, + "text_fingerprint": "3364a7396e2c62b05bce5f42c49fc9697859159b21dbe384d9932e592de43e9f", + "type": "기타정상" + } + ], + "lowest_scoring_phishing": [ + { + "probability": 0.09207301406592294, + "text_fingerprint": "829fea59062a71258522b9c5cc465d68b271418a21b667825e9e41e699928603", + "type": "지인가족사칭" + }, + { + "probability": 0.09219815756108402, + "text_fingerprint": "2717c495b40216b748632e4da0eb0a30dc073b9c8a77582aaab21ac0b7ea1539", + "type": "채용부업사기" + }, + { + "probability": 0.14155572598290803, + "text_fingerprint": "a81efa689e5bab91577bbfd3c0ae3e32a941d3c8c7b18566cc15e6abf4479070", + "type": "투자리딩방사기" + }, + { + "probability": 0.16069060704267482, + "text_fingerprint": "4f497e918721762642d62776843e758fe66b7c594b937b4c16fee9697f49e56b", + "type": "금융기관사칭" + }, + { + "probability": 0.2105938142203876, + "text_fingerprint": "3a6048f2cc45147d555c7416de1fbf811a438bb9ddb9b057ab2d38fead39b9c0", + "type": "투자리딩방사기" + }, + { + "probability": 0.21449347096101068, + "text_fingerprint": "6042b81b11cfa5a8c9db730e483d9ca119afda3caa0a9e2831fa3ca066689d92", + "type": "지인가족사칭" + }, + { + "probability": 0.24844427303506378, + "text_fingerprint": "61ccd047a6d5f78a71f945165a65d40f04f8c92424d699d6319c5af94682c7fb", + "type": "택배배송사칭" + }, + { + "probability": 0.27662465551928267, + "text_fingerprint": "2fbc6b4e11113b0d39ced22dcfda822d3b64760260a479766403efa1ef49e591", + "type": "지인가족사칭" + }, + { + "probability": 0.3014322884198851, + "text_fingerprint": "9a17e496aea032ec4edc7355bfa89673f14a14bd017bec19bdc0c316cacd41b4", + "type": "투자리딩방사기" + }, + { + "probability": 0.35775545908217515, + "text_fingerprint": "46335873d6264088f204441052209785487588dec4113a6a4bcbfb7e62fe15ff", + "type": "금융기관사칭" + }, + { + "probability": 0.3676898135052793, + "text_fingerprint": "d5b9a5c7d227f2f6a0dbe6d268581389963019893ce9564553a310334234586f", + "type": "금융기관사칭" + }, + { + "probability": 0.3805884399469613, + "text_fingerprint": "5cac45fd0afdc096318548b3a06032cd938689d86025d9e81e2e3fc76e14b33f", + "type": "금융기관사칭" + }, + { + "probability": 0.38143409353406227, + "text_fingerprint": "cf3a5f447f417e56a6acd365134a427a10ac98f7a9b2566b55ef7ff3a82b8cb4", + "type": "금융기관사칭" + }, + { + "probability": 0.39076470684775166, + "text_fingerprint": "e748c7274d6ce8ee555f9f8cd5e86a67927ef282c831ed5ace423f7e7a7a4cc1", + "type": "지인가족사칭" + }, + { + "probability": 0.41087584191368753, + "text_fingerprint": "41f5bb6108ef76446fbdb8e19be660b9ff53d89b9e9d1b5e90847deae2c1a24e", + "type": "금융기관사칭" + } + ] + }, + "false_positives_at_artifact_threshold": { + "by_type": { + "기타정상": { + "false_positive": 2, + "false_positive_rate": 0.15384615384615385, + "sample_count": 13 + }, + "일상대화": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 34 + }, + "정상광고프로모션": { + "false_positive": 6, + "false_positive_rate": 0.75, + "sample_count": 8 + }, + "정상금융알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 6 + }, + "정상카드결제알림": { + "false_positive": 1, + "false_positive_rate": 0.25, + "sample_count": 4 + }, + "정상택배배송안내": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 6 + }, + "정상포인트소멸알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "sample_count": 3 + } + }, + "false_positive_count": 9, + "false_positive_rate": 0.12162162162162163, + "normal_count": 74, + "threshold": 0.3038914498086088 + }, + "normal_count": 74, + "operating_points": [ + { + "max_recall": 0.5662650602409639, + "missed_phishing": 36, + "target_fpr": 0.02, + "threshold": 0.8118198057976024 + }, + { + "max_recall": 0.6987951807228916, + "missed_phishing": 25, + "target_fpr": 0.05, + "threshold": 0.6426829026388645 + }, + { + "max_recall": 0.8192771084337349, + "missed_phishing": 15, + "target_fpr": 0.1, + "threshold": 0.46235980810013594 + }, + { + "max_recall": 0.9156626506024096, + "missed_phishing": 7, + "target_fpr": 0.15, + "threshold": 0.27662465551928267 + }, + { + "max_recall": 0.927710843373494, + "missed_phishing": 6, + "target_fpr": 0.2, + "threshold": 0.24844427303506378 + } + ], + "phishing_count": 83, + "pr_auc": 0.9502509951841229, + "roc_auc": 0.9366655812438944, + "sample_count": 157, + "split": "real_holdout" + } + ], + "structural_feature_names": [ + "naive_bayes_score", + "logistic_regression_score", + "linear_svm_score", + "has_url", + "has_short_url", + "has_phone", + "has_account", + "has_card", + "has_amount", + "has_web_tag", + "has_urgency", + "has_transfer_request", + "has_personal_info_request", + "has_link_action", + "is_long_text", + "has_ad_disclosure", + "has_opt_out" + ] +} diff --git a/data_science/SMSModel/reports/stacking_v3-baseline/test_evaluation.json b/data_science/SMSModel/reports/stacking_v3-baseline/test_evaluation.json new file mode 100644 index 0000000..1d90dd1 --- /dev/null +++ b/data_science/SMSModel/reports/stacking_v3-baseline/test_evaluation.json @@ -0,0 +1,359 @@ +{ + "error_samples": { + "false_negatives": [ + { + "text_fingerprint": "69f97bb6c8973d864ca30434d19ec6e7fa9ff2900b017c07d47f4af10b43fff3" + }, + { + "text_fingerprint": "fd2d19d83512b7149a8be03805c12950e64fe7db3708134956cac385580a37ab" + }, + { + "text_fingerprint": "f486cca20e58cf0e8ab98706d819579adab277a57e29852b607f948e94e6e715" + }, + { + "text_fingerprint": "10473b8eb05ba7f1a3c9e53b8f198a1eabf368220622ab34f18a144595a3dc28" + } + ], + "false_positives": [ + { + "text_fingerprint": "dd9c13a2c2f2747da2a12764030c3b1df5dbeb94dde8116dd4b4cdf9e93df172" + }, + { + "text_fingerprint": "5580c99edd3218684721ed377d478d39496755e1956427c69ec48ccedee48db8" + }, + { + "text_fingerprint": "e142cdd243127df2be02681d70ab63c0727d0f5ea21b8fe01e32aac6f0a5f7d6" + }, + { + "text_fingerprint": "a5bd16e9e8c638a07feac44cc91021ad0b5b5a56523b4f7a52050065625e0907" + }, + { + "text_fingerprint": "7e3aabb364ce57866b881da6ef7dd6b953268e05267a323c72a50799df39731f" + }, + { + "text_fingerprint": "95adda6bb56eadb664fafd78110e4ad88f22880471679ffb558176b6da1c3153" + }, + { + "text_fingerprint": "4179f54dfe26365994664664a8621684d1b47818b3588d81bd890449acf218ee" + }, + { + "text_fingerprint": "3ffe7351bbaae187b623b7d3f36883ce33291d3bd21776a744e1ce49899884be" + }, + { + "text_fingerprint": "4500e37aa1d42963021b06531c284b87f20d0c6f558084f04a52a9a34fabb42c" + }, + { + "text_fingerprint": "ccce2b7f00a16f8446b78440ca74d229f7f29faf08ae1b8b34ddebc43bee98fb" + }, + { + "text_fingerprint": "c93b8bcf3754cd8d9921df04f274c9bbadd47527e802113dd4751ddbd96ad7bd" + }, + { + "text_fingerprint": "dc3b31cf3ae71d5c7accc6ac74255783ae77a04394219d290d1b53f7d791213c" + }, + { + "text_fingerprint": "e14cafbff49aa7a3e2397a2ede7aeef4ff0003e2a3a9c144b1139a0597211374" + } + ] + }, + "latency_stats": { + "mean": 7.699250427350429, + "p50": 7.5386, + "p95": 10.65416, + "sample_count": 117 + }, + "normal_by_type": { + "기타정상": { + "false_positive": 2, + "false_positive_rate": 0.4, + "false_positive_rate_95_ci": { + "lower": 0.1176182311592533, + "upper": 0.769280067791163 + }, + "sample_count": 5, + "true_negative": 3 + }, + "일상대화": { + "false_positive": 0, + "false_positive_rate": 0.0, + "false_positive_rate_95_ci": { + "lower": 0.0, + "upper": 0.15464382326420195 + }, + "sample_count": 21, + "true_negative": 21 + }, + "정상공공기관알림": { + "false_positive": 3, + "false_positive_rate": 0.75, + "false_positive_rate_95_ci": { + "lower": 0.3006360524426366, + "upper": 0.9544139373553637 + }, + "sample_count": 4, + "true_negative": 1 + }, + "정상광고프로모션": { + "false_positive": 5, + "false_positive_rate": 0.625, + "false_positive_rate_95_ci": { + "lower": 0.30573785458380187, + "upper": 0.863158240538479 + }, + "sample_count": 8, + "true_negative": 3 + }, + "정상금융알림": { + "false_positive": 1, + "false_positive_rate": 0.25, + "false_positive_rate_95_ci": { + "lower": 0.045586062644636216, + "upper": 0.6993639475573634 + }, + "sample_count": 4, + "true_negative": 3 + }, + "정상인증알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "false_positive_rate_95_ci": { + "lower": 0.0, + "upper": 0.48990002040399916 + }, + "sample_count": 4, + "true_negative": 4 + }, + "정상카드결제알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "false_positive_rate_95_ci": { + "lower": 0.0, + "upper": 0.48990002040399916 + }, + "sample_count": 4, + "true_negative": 4 + }, + "정상택배배송안내": { + "false_positive": 2, + "false_positive_rate": 0.3333333333333333, + "false_positive_rate_95_ci": { + "lower": 0.09676933255921683, + "upper": 0.7000116786584712 + }, + "sample_count": 6, + "true_negative": 4 + }, + "정상포인트소멸알림": { + "false_positive": 0, + "false_positive_rate": 0.0, + "false_positive_rate_95_ci": { + "lower": 0.0, + "upper": 0.6576280471103807 + }, + "sample_count": 2, + "true_negative": 2 + } + }, + "overall_metrics": { + "accuracy": 0.8547008547008547, + "confusion_matrix": { + "false_negative": 4, + "false_positive": 13, + "true_negative": 45, + "true_positive": 55 + }, + "f1_score": 0.8661417322834646, + "f2_score": 0.9046052631578947, + "precision": 0.8088235294117647, + "recall": 0.9322033898305084 + }, + "phishing_by_type": { + "결제환불사칭": { + "false_negative": 0, + "recall": 1.0, + "recall_95_ci": { + "lower": 0.20654329147389294, + "upper": 1.0 + }, + "sample_count": 1, + "true_positive": 1 + }, + "경조사사칭": { + "false_negative": 0, + "recall": 1.0, + "recall_95_ci": { + "lower": 0.7224598312333834, + "upper": 1.0 + }, + "sample_count": 10, + "true_positive": 10 + }, + "계정정지본인인증유도": { + "false_negative": 0, + "recall": 1.0, + "recall_95_ci": { + "lower": 0.20654329147389294, + "upper": 1.0 + }, + "sample_count": 1, + "true_positive": 1 + }, + "금융기관사칭": { + "false_negative": 0, + "recall": 1.0, + "recall_95_ci": { + "lower": 0.7961107336956521, + "upper": 1.0 + }, + "sample_count": 15, + "true_positive": 15 + }, + "기타피싱": { + "false_negative": 0, + "recall": 1.0, + "recall_95_ci": { + "lower": 0.20654329147389294, + "upper": 1.0 + }, + "sample_count": 1, + "true_positive": 1 + }, + "대출사기": { + "false_negative": 0, + "recall": 1.0, + "recall_95_ci": { + "lower": 0.20654329147389294, + "upper": 1.0 + }, + "sample_count": 1, + "true_positive": 1 + }, + "복합피싱": { + "false_negative": 0, + "recall": 1.0, + "recall_95_ci": { + "lower": 0.20654329147389294, + "upper": 1.0 + }, + "sample_count": 1, + "true_positive": 1 + }, + "악성링크앱설치유도": { + "false_negative": 1, + "recall": 0.0, + "recall_95_ci": { + "lower": 0.0, + "upper": 0.7934567085261071 + }, + "sample_count": 1, + "true_positive": 0 + }, + "이벤트당첨사칭": { + "false_negative": 0, + "recall": 1.0, + "recall_95_ci": { + "lower": 0.7846829880728186, + "upper": 1.0 + }, + "sample_count": 14, + "true_positive": 14 + }, + "정부공공기관사칭": { + "false_negative": 1, + "recall": 0.75, + "recall_95_ci": { + "lower": 0.3006360524426366, + "upper": 0.9544139373553637 + }, + "sample_count": 4, + "true_positive": 3 + }, + "중고거래사기": { + "false_negative": 0, + "recall": 1.0, + "recall_95_ci": { + "lower": 0.20654329147389294, + "upper": 1.0 + }, + "sample_count": 1, + "true_positive": 1 + }, + "지인가족사칭": { + "false_negative": 1, + "recall": 0.0, + "recall_95_ci": { + "lower": 0.0, + "upper": 0.7934567085261071 + }, + "sample_count": 1, + "true_positive": 0 + }, + "채용부업사기": { + "false_negative": 0, + "recall": 1.0, + "recall_95_ci": { + "lower": 0.20654329147389294, + "upper": 1.0 + }, + "sample_count": 1, + "true_positive": 1 + }, + "택배배송사칭": { + "false_negative": 0, + "recall": 1.0, + "recall_95_ci": { + "lower": 0.6096569663469354, + "upper": 0.9999999999999999 + }, + "sample_count": 6, + "true_positive": 6 + }, + "투자리딩방사기": { + "false_negative": 1, + "recall": 0.0, + "recall_95_ci": { + "lower": 0.0, + "upper": 0.7934567085261071 + }, + "sample_count": 1, + "true_positive": 0 + } + }, + "sample_summary": { + "failure_counts": { + "engine_failure": 0, + "exception": 0, + "missing_result": 0 + }, + "successful_samples": 117, + "total_samples": 117 + }, + "schema_version": 1, + "shared_metrics": { + "accuracy": 0.8547008547008547, + "f1": 0.8661417322834646, + "f2": 0.9046052631578947, + "false_negative": 4, + "false_positive": 13, + "precision": 0.8088235294117647, + "recall": 0.9322033898305084, + "sample_count": 117, + "true_negative": 45, + "true_positive": 55 + }, + "split": "test", + "threshold": 0.30448047609435736, + "unavailable_models": [], + "unique_template_metrics": { + "accuracy": 0.8433734939759037, + "f1_score": 0.8266666666666667, + "f2_score": 0.861111111111111, + "false_negative": 4, + "false_positive": 9, + "precision": 0.775, + "recall": 0.8857142857142857, + "sample_count": 83, + "true_negative": 39, + "true_positive": 31 + } +} diff --git a/data_science/SMSModel/reports/stacking_v3-baseline/test_evaluation.md b/data_science/SMSModel/reports/stacking_v3-baseline/test_evaluation.md new file mode 100644 index 0000000..362b238 --- /dev/null +++ b/data_science/SMSModel/reports/stacking_v3-baseline/test_evaluation.md @@ -0,0 +1,19 @@ +# Stacking v3 Test Evaluation + +Threshold selection: validation only; final metrics: test only. + +| Metric | Value | +|---|---:| +| Samples | 117 | +| Accuracy | 0.854701 | +| Precision | 0.808824 | +| Recall | 0.932203 | +| F1 | 0.866142 | +| F2 | 0.904605 | +| TN | 45 | +| FP | 13 | +| FN | 4 | +| TP | 55 | +| Mean latency (ms) | 7.699 | +| P50 latency (ms) | 7.539 | +| P95 latency (ms) | 10.654 | diff --git a/data_science/SMSModel/reports/stacking_v3/test_evaluation.json b/data_science/SMSModel/reports/stacking_v3/test_evaluation.json index 1d90dd1..53dd60c 100644 --- a/data_science/SMSModel/reports/stacking_v3/test_evaluation.json +++ b/data_science/SMSModel/reports/stacking_v3/test_evaluation.json @@ -27,6 +27,9 @@ { "text_fingerprint": "a5bd16e9e8c638a07feac44cc91021ad0b5b5a56523b4f7a52050065625e0907" }, + { + "text_fingerprint": "f9cf49989bd91792719aa506fa701db83db29c35863fcb7e555ca854d14ba637" + }, { "text_fingerprint": "7e3aabb364ce57866b881da6ef7dd6b953268e05267a323c72a50799df39731f" }, @@ -39,6 +42,12 @@ { "text_fingerprint": "3ffe7351bbaae187b623b7d3f36883ce33291d3bd21776a744e1ce49899884be" }, + { + "text_fingerprint": "b7d667bc51f5c9fe97cb257c1e513e5f019c61be26f6771dfe8843883a5bf29d" + }, + { + "text_fingerprint": "1b538383bd482eac3211546f3eb77fe3340f71e1a507f1727bbcd7c5554e4ba7" + }, { "text_fingerprint": "4500e37aa1d42963021b06531c284b87f20d0c6f558084f04a52a9a34fabb42c" }, @@ -57,9 +66,9 @@ ] }, "latency_stats": { - "mean": 7.699250427350429, - "p50": 7.5386, - "p95": 10.65416, + "mean": 6.667278632478633, + "p50": 6.5228, + "p95": 8.33592, "sample_count": 117 }, "normal_by_type": { @@ -94,14 +103,14 @@ "true_negative": 1 }, "정상광고프로모션": { - "false_positive": 5, - "false_positive_rate": 0.625, + "false_positive": 8, + "false_positive_rate": 1.0, "false_positive_rate_95_ci": { - "lower": 0.30573785458380187, - "upper": 0.863158240538479 + "lower": 0.6755843804891231, + "upper": 1.0 }, "sample_count": 8, - "true_negative": 3 + "true_negative": 0 }, "정상금융알림": { "false_positive": 1, @@ -155,16 +164,16 @@ } }, "overall_metrics": { - "accuracy": 0.8547008547008547, + "accuracy": 0.8290598290598291, "confusion_matrix": { "false_negative": 4, - "false_positive": 13, - "true_negative": 45, + "false_positive": 16, + "true_negative": 42, "true_positive": 55 }, - "f1_score": 0.8661417322834646, - "f2_score": 0.9046052631578947, - "precision": 0.8088235294117647, + "f1_score": 0.846153846153846, + "f2_score": 0.8957654723127035, + "precision": 0.7746478873239436, "recall": 0.9322033898305084 }, "phishing_by_type": { @@ -330,19 +339,19 @@ }, "schema_version": 1, "shared_metrics": { - "accuracy": 0.8547008547008547, - "f1": 0.8661417322834646, - "f2": 0.9046052631578947, + "accuracy": 0.8290598290598291, + "f1": 0.8461538461538461, + "f2": 0.8957654723127035, "false_negative": 4, - "false_positive": 13, - "precision": 0.8088235294117647, + "false_positive": 16, + "precision": 0.7746478873239436, "recall": 0.9322033898305084, "sample_count": 117, - "true_negative": 45, + "true_negative": 42, "true_positive": 55 }, "split": "test", - "threshold": 0.30448047609435736, + "threshold": 0.3038914498086088, "unavailable_models": [], "unique_template_metrics": { "accuracy": 0.8433734939759037, diff --git a/data_science/SMSModel/reports/stacking_v3/test_evaluation.md b/data_science/SMSModel/reports/stacking_v3/test_evaluation.md index 362b238..bbf4b94 100644 --- a/data_science/SMSModel/reports/stacking_v3/test_evaluation.md +++ b/data_science/SMSModel/reports/stacking_v3/test_evaluation.md @@ -5,15 +5,15 @@ Threshold selection: validation only; final metrics: test only. | Metric | Value | |---|---:| | Samples | 117 | -| Accuracy | 0.854701 | -| Precision | 0.808824 | +| Accuracy | 0.829060 | +| Precision | 0.774648 | | Recall | 0.932203 | -| F1 | 0.866142 | -| F2 | 0.904605 | -| TN | 45 | -| FP | 13 | +| F1 | 0.846154 | +| F2 | 0.895765 | +| TN | 42 | +| FP | 16 | | FN | 4 | | TP | 55 | -| Mean latency (ms) | 7.699 | -| P50 latency (ms) | 7.539 | -| P95 latency (ms) | 10.654 | +| Mean latency (ms) | 6.667 | +| P50 latency (ms) | 6.523 | +| P95 latency (ms) | 8.336 | diff --git a/data_science/SMSModel/run_error_analysis.py b/data_science/SMSModel/run_error_analysis.py new file mode 100644 index 0000000..c97d076 --- /dev/null +++ b/data_science/SMSModel/run_error_analysis.py @@ -0,0 +1,226 @@ +"""오탐·미탐 원인 분석""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import joblib +import numpy as np +import pandas as pd +from sklearn.metrics import average_precision_score, roc_auc_score, roc_curve + +from data_science.SMSModel.train_sms import ( + DATA_PATH, + load_data, + select_real_holdout, + split_data, +) + +SMS_MODEL_DIRECTORY = Path(__file__).resolve().parent +DEFAULT_MODEL_PATH = ( + SMS_MODEL_DIRECTORY / "artifacts" / "stacking" / "v3" / "model.joblib" +) +DEFAULT_OUTPUT_PATH = SMS_MODEL_DIRECTORY / "reports" / "error_analysis.json" + +# 오탐률 곡선을 확인할 지점 +FPR_TARGETS = (0.02, 0.05, 0.10, 0.15, 0.20) + +# 원인 파악용으로 들여다볼 표본 수 +TOP_SAMPLE_COUNT = 15 + + +def load_classifier(model_path: Path): + """artifact에서 분류기 꺼냄""" + payload = joblib.load(model_path) + if isinstance(payload, dict) and "classifier" in payload: + return payload["classifier"] + return payload + + +def score_frame(classifier, frame: pd.DataFrame) -> pd.DataFrame: + """확률을 붙인 사본을 반환""" + probabilities, unavailable = classifier.predict_probabilities(frame) + if unavailable: + raise RuntimeError(f"base models unavailable: {unavailable}") + + scored = frame.copy() + scored["probability"] = np.asarray(probabilities, dtype=float) + return scored + + +def summarize_operating_points(scored: pd.DataFrame) -> list[dict[str, float]]: + """목표 오탐률마다 도달 가능한 최대 Recall을 계산""" + truth = (scored["label"].to_numpy() == "phishing").astype(int) + probability = scored["probability"].to_numpy() + + false_positive_rate, true_positive_rate, thresholds = roc_curve( + truth, probability + ) + + points = [] + for target in FPR_TARGETS: + feasible = np.where(false_positive_rate <= target)[0] + if feasible.size == 0: + continue + index = int(feasible.max()) + points.append( + { + "target_fpr": target, + "max_recall": float(true_positive_rate[index]), + "threshold": float(thresholds[index]), + "missed_phishing": int( + round((1 - true_positive_rate[index]) * truth.sum()) + ), + } + ) + return points + + +def summarize_false_positives( + scored: pd.DataFrame, + threshold: float, +) -> dict[str, object]: + """임계값을 넘은 정상 문자를 유형별로 집계""" + normals = scored[scored["label"] == "normal"] + flagged_total = int((normals["probability"] >= threshold).sum()) + + by_type: dict[str, dict[str, float]] = {} + for type_name, group in normals.groupby("type"): + flagged = int((group["probability"] >= threshold).sum()) + by_type[str(type_name)] = { + "sample_count": int(len(group)), + "false_positive": flagged, + "false_positive_rate": flagged / len(group), + } + + return { + "threshold": float(threshold), + "normal_count": int(len(normals)), + "false_positive_count": flagged_total, + "false_positive_rate": ( + flagged_total / len(normals) if len(normals) else 0.0 + ), + "by_type": dict( + sorted( + by_type.items(), + key=lambda item: -item[1]["false_positive_rate"], + ) + ), + } + + +def list_extreme_samples(scored: pd.DataFrame) -> dict[str, list[dict]]: + """임계값을 밀어올리는 정상과, 놓치기 쉬운 피싱 추출""" + + def to_records(frame: pd.DataFrame) -> list[dict]: + """원문 없이 지문·유형·확률만 남긴 표본 레코드로 변환""" + return [ + { + "text_fingerprint": str(row.get("text_fingerprint", "")), + "type": str(row["type"]), + "probability": float(row["probability"]), + } + for _, row in frame.iterrows() + ] + + normals = scored[scored["label"] == "normal"] + phishing = scored[scored["label"] == "phishing"] + + return { + # 확률이 높은 정상 = 임계값을 위로 미는 표본 + "highest_scoring_normals": to_records( + normals.nlargest(TOP_SAMPLE_COUNT, "probability") + ), + # 확률이 낮은 피싱 = 임계값을 올리면 놓치는 표본 + "lowest_scoring_phishing": to_records( + phishing.nsmallest(TOP_SAMPLE_COUNT, "probability") + ), + } + + +def analyze(classifier, name: str, frame: pd.DataFrame) -> dict[str, object]: + """split 하나에 대한 분석 결과 생성""" + scored = score_frame(classifier, frame) + truth = (scored["label"].to_numpy() == "phishing").astype(int) + probability = scored["probability"].to_numpy() + + return { + "split": name, + "sample_count": int(len(scored)), + "phishing_count": int(truth.sum()), + "normal_count": int(len(truth) - truth.sum()), + "roc_auc": float(roc_auc_score(truth, probability)), + "pr_auc": float(average_precision_score(truth, probability)), + "operating_points": summarize_operating_points(scored), + "false_positives_at_artifact_threshold": summarize_false_positives( + scored, classifier.threshold + ), + "extreme_samples": list_extreme_samples(scored), + } + + +def build_report(classifier) -> dict[str, object]: + """validation·test·real_holdout 분석을 한데 모으기""" + pool, holdout = load_data(DATA_PATH) + splits = split_data(pool, create_manifest=False) + + return { + "artifact_threshold": float(classifier.threshold), + "structural_feature_names": list( + classifier.get_metadata()["meta_feature_names"] + ), + "splits": [ + analyze(classifier, "validation", splits.validation), + analyze(classifier, "test", splits.test), + analyze(classifier, "real_holdout", select_real_holdout(holdout)), + ], + } + + +def main() -> None: + """CLI 인자를 읽어 오탐·미탐 분석 리포트를 JSON으로 저장""" + + parser = argparse.ArgumentParser( + description="Analyze false positives and missed phishing." + ) + parser.add_argument("--model-path", type=Path, default=DEFAULT_MODEL_PATH) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT_PATH) + arguments = parser.parse_args() + + classifier = load_classifier(arguments.model_path) + report = build_report(classifier) + + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + print(f"[Error analysis] {arguments.output}") + print(f"[Error analysis] threshold={report['artifact_threshold']:.6f}") + + for split in report["splits"]: + print( + f" {split['split']:<14} " + f"n={split['sample_count']:<4} " + f"ROC-AUC {split['roc_auc']:.4f}" + ) + for point in split["operating_points"]: + if point["target_fpr"] == 0.05: + print( + f" FPR 5% → Recall {point['max_recall']:.4f} " + f"(피싱 {point['missed_phishing']}건 놓침)" + ) + false_positives = split["false_positives_at_artifact_threshold"] + worst = list(false_positives["by_type"].items())[:3] + for type_name, stats in worst: + if stats["false_positive"]: + print( + f" FP {stats['false_positive']}/{stats['sample_count']}" + f" {type_name}" + ) + + +if __name__ == "__main__": + main() diff --git a/docs/STACKING_V3_FP_IMPROVEMENT.md b/docs/STACKING_V3_FP_IMPROVEMENT.md new file mode 100644 index 0000000..18182f1 --- /dev/null +++ b/docs/STACKING_V3_FP_IMPROVEMENT.md @@ -0,0 +1,210 @@ +# Stacking v3 오탐 개선 실험 보고서 + +Issue #84 · Stacking v3 재학습 및 유형별 FP/FN 개선 검증 + +## 1. 목적과 결론 + +`#83`에서 데이터셋을 정비한 뒤에도 실제 문자 평가셋의 정상 오탐률이 14.86%로 남았다. 이 실험은 오탐이 어디에 집중돼 있는지 특정하고, 구조적 특징으로 해소할 수 있는지 판정한다. + +오탐은 **광고성 정상 문자에 집중**돼 있었다. 법정 광고 표기를 구조적 특징으로 추가한 결과, 표기가 있는 광고에서는 오탐이 완전히 사라졌고 **퇴행은 한 건도 없었다**. 다만 사전에 정한 두 관문은 모두 미달했다. + +| 관문 | 기준 | 결과 | 판정 | +|---|---|---:|---| +| 1차 | real_holdout ROC-AUC ≥ 0.95 | 0.9367 | 미달 | +| 2차 | 정상 FPR 5%에서 Recall ≥ 0.85 | 0.6988 | 미달 | +| 3차 | 광고 FP 유의 감소 | validation p=0.0005 | 통과 | + +**결론: 특징 자체는 유효하나 단독 운영 기준에는 미달한다.** 미달의 주된 원인은 모델이 아니라 학습 데이터의 중복 구조로 확인됐다(5장). + +## 2. 오탐 원인 분석 + +특징 추가 전 artifact(`v3-baseline`, threshold 0.304480)로 split별 오탐을 집계했다. + +| split | 표본 | ROC-AUC | 정상 오탐률 | FPR 5%에서 Recall | +|---|---:|---:|---:|---:| +| validation | 122 | 0.8895 | 32.8% | 0.6897 | +| test | 117 | 0.9643 | 22.4% | 0.9153 | +| real_holdout | 157 | 0.9357 | 14.9% | 0.5783 | + +### 2.1 오탐이 광고 문자에 집중된다 + +정상 유형별 오탐 건수다. + +| 유형 | validation | test | real_holdout | +|---|---|---|---| +| **정상광고프로모션** | **12/12** | 5/8 | **8/8** | +| 기타정상 | 3/5 | 2/5 | 2/13 | +| 정상공공기관알림 | 2/5 | 3/4 | — | +| 정상금융알림 | 2/5 | 1/4 | — | +| 정상택배배송안내 | 2/5 | 2/6 | — | +| 정상카드결제알림 | — | — | 1/4 | + +`정상광고프로모션`은 validation과 real_holdout에서 **오탐률 100%** 였다. 광고 문자를 단 한 건도 정상으로 판정하지 못했다. + +확률이 가장 높은 정상 문자도 광고가 지배적이었다(롯데월드 0.869 · 배달의민족 0.858 · 부메랑 쿠폰 0.751 · 근로장려금 광고 0.695). 이들이 임계값을 위로 밀어올려 피싱 Recall을 함께 떨어뜨리고 있었다. + +### 2.2 반대편은 문제가 아니었다 + +정상 확률 중앙값보다 낮은 피싱은 validation 2건, real_holdout 1건뿐이었다. **놓치는 피싱이 많아서가 아니라, 정상을 과하게 걸러서** 운영점이 나빠지는 구조였다. + +## 3. 구조적 특징 추가 + +### 3.1 근거 + +정보통신망법상 영리 목적 광고성 정보에는 `(광고)` 표기와 수신거부 방법을 명시해야 한다. 피싱은 이 표기를 갖추지 않거나 흉내만 내는 경우가 많다. + +데이터셋 전체에서 보유율을 측정했다. + +| 대상 | 표기 보유 | 비율 | +|---|---:|---:| +| 정상 | 33 / 546 | 6.0% | +| 피싱 | 14 / 538 | 2.6% | +| **정상광고프로모션** | **31 / 42** | **73.8%** | + +표기가 있는 문자가 정상일 확률은 70.2%다. 완전한 판별자는 아니므로 **하드 규칙이 아닌 특징**으로 추가했다. + +### 3.2 구현 + +`app/analysis/text/structural_features.py`에 패턴 두 개와 특징 두 개를 추가했다. + +```text +AD_DISCLOSURE_PATTERN (광고) · [광고] · 문두 광고 +OPT_OUT_PATTERN 무료수신거부 · 수신거부 · 080 번호 + +has_ad_disclosure, has_opt_out → 구조 특징 12개에서 14개로 +``` + +`STACKING_STRUCTURAL_FEATURE_NAMES`는 학습과 추론에서 열 순서가 같아야 하므로 **기존 열 뒤에만** 덧붙였다. 새 특징이 항상 마지막에 오는지 검증하는 테스트를 함께 추가했다. + +특징 수가 바뀌면 meta-classifier 입력 폭이 달라져 기존 artifact를 재사용할 수 없다. 비교를 위해 특징 추가 전 artifact를 `artifacts/stacking/v3-baseline`으로 보존했다. + +## 4. 특징 추가 전후 비교 + +선택된 threshold는 0.304480에서 0.303891로 거의 변하지 않았다. 데이터를 바꾸지 않았으므로 아래 차이는 특징 추가만의 효과다. + +### 4.1 전체 지표 + +| split | ROC-AUC | FPR 5% Recall | 정상 오탐률 | +|---|---|---|---| +| validation | 0.8895 → **0.9448** | 0.6897 → 0.6897 | 32.8% → **14.1%** | +| test | 0.9643 → 0.9570 | 0.9153 → 0.8983 | 22.4% → 27.6% | +| real_holdout | 0.9357 → 0.9367 | 0.5783 → **0.6988** | 14.9% → **12.2%** | + +real_holdout에서 오탐률 5% 지점에 놓치는 피싱이 **35건에서 25건으로** 줄었다. + +### 4.2 McNemar exact test + +| split | 둘 다 정답 | 둘 다 오답 | baseline만 정답 | 신규만 정답 | 정확도 | p-value | +|---|---:|---:|---:|---:|---|---:| +| validation | 99 | 11 | **0** | **12** | 0.8115 → 0.9098 | **0.0005** | +| test | 97 | 17 | 3 | 0 | 0.8547 → 0.8291 | 0.2500 | +| real_holdout | 137 | 18 | **0** | 2 | 0.8726 → 0.8854 | 0.5000 | + +**validation과 real_holdout에서 "baseline만 정답"이 0건이다.** 특징 추가로 새로 틀린 표본이 없다는 뜻이며, 개선만 있고 퇴행이 없다. + +validation의 개선은 통계적으로 유의하다(p=0.0005). 개선된 12건은 정확히 광고 문자 12건이다. + +### 4.3 광고 문자 오탐 + +| split | 전 | 후 | +|---|---|---| +| validation | 12/12 | **0/12** | +| test | 5/8 | 8/8 | +| real_holdout | 8/8 | **6/8** | + +test만 악화된 이유는 표기 보유 여부로 갈린다. + +| split | 광고 표본 | 표기 보유 | 표기 있는 광고 오탐 | 표기 없는 광고 오탐 | +|---|---:|---:|---|---| +| train | 14 | 12 (86%) | **0/12** | 1/2 | +| validation | 12 | 12 (100%) | **0/12** | — | +| test | 8 | **0 (0%)** | — | 8/8 | +| real_holdout | 8 | 7 (88%) | 5/7 | 1/1 | + +**표기가 있는 광고에서는 train과 validation 모두 오탐이 0건이다.** 특징은 의도대로 작동했다. test가 나빠 보인 것은 그 8건에 표기가 하나도 없기 때문이다. + +### 4.4 추론 성능 + +| artifact | P50 | P95 | 평균 | +|---|---:|---:|---:| +| baseline | 7.539 ms | 10.654 ms | 7.699 ms | +| 특징 추가 후 | 6.523 ms | 8.336 ms | 6.667 ms | + +두 artifact 모두 base model 가용성 문제는 없었다(`unavailable_models` 비어 있음). + +## 5. 미달 원인: 학습 데이터의 중복 구조 + +test의 광고 8건은 **금액만 다른 동일 문장**이었다. + +```text +환급금 분석결과 [Web발신] 박*수님 - 분석결과: 환급 가능성 발견 +- 분석근거: 박*수님의 해당사항 3개 - 예상환급액: [금액] (제휴사 환급신고액 평균) +``` + +금액을 마스킹하면 고유 문장은 1개이고, 8건 모두 같은 template group(`tpl_1b538383bd48`)에 속한다. 이 그룹이 통째로 test에 배정되면서 test의 광고 표본이 전부 표기 없는 문자가 됐고, train에는 표기 없는 광고가 2건만 남았다. + +라벨 자체는 타당하다. 환급 대행 서비스 마케팅 문자로 판단되며 이진 라벨을 변경하지 않았다. + +### 5.1 같은 문제가 데이터셋 전반에 있다 + +숫자만 바꾼 변형이 3건 이상인 template group을 조사했다. + +| 행수 | label / 유형 | 문장 | +|---:|---|---| +| 19 | phishing / 지인가족사칭 | "장모님 저 급히 송금할데가… [금액]이에요" | +| 16 | normal / 기타정상 | "오늘 당일 하루! 아르바이트 모집… 일급 [금액]" | +| 14 | phishing / 채용부업사기 | "취업비서 인쿠르트의 김[N]입니다…" | +| 13 | phishing / 금융기관사칭 | "마스크하고 손소독제를 싸게… [금액] 보내줘" | +| 9 · 8 | phishing / 지인가족사칭 | "형수님 진짜 죄송한데 일본에…" | +| 8 | normal / 정상광고프로모션 | 환급금 분석결과 | +| 4 · 4 · 3 · 3 | normal / 정상카드결제알림 | 소비쿠폰 사용 알림 | + +**11개 그룹 101행이며 전체 1,084행의 9.3%다.** 피싱 63행, 정상 38행으로 양쪽 클래스에 걸쳐 있다. + +`#77`이 명시한 원칙에 어긋난다. + +> 동일 문장의 숫자나 URL만 변경해 데이터 개수를 늘리는 방식은 지양한다. + +fingerprint 중복 제거는 숫자가 달라 걸러내지 못하고, template 그룹화는 같은 split에 묶어둘 뿐 행 수를 줄이지 않는다. + +이 실험에서는 정정하지 않았다. 데이터를 동시에 바꾸면 특징 추가의 효과를 분리해 측정할 수 없기 때문이다. **후속 이슈로 분리한다.** + +## 6. 판정과 다음 조치 + +### 판정 + +1차·2차 관문 미달, 3차 관문 통과. 현재 artifact는 단독 운영 후보로 채택하지 않는다. + +### 확인된 사실 + +1. 오탐은 광고성 정상 문자에 집중돼 있었고, 법정 광고 표기 특징으로 **표기가 있는 경우 완전히 해소**된다. +2. 특징 추가로 인한 **퇴행은 없다**(validation·real_holdout에서 "baseline만 정답" 0건). +3. 남은 오탐의 상당 부분은 모델이 아니라 **학습 데이터의 중복 구조** 때문이다. + +### 다음 조치 + +1. **숫자 변형 중복 정리** — 11개 그룹 101행을 대표 표본으로 축소한다. `#77` 원칙 준수이자 이번 미달의 직접 원인 해소다. +2. **표기 없는 광고 표본 확보** — 현재 실질 4종류뿐이라 이 구간을 학습할 근거가 없다. 다만 `#83`에서 확인했듯 합성 생성은 역효과이므로 실제 수집이 필요하다. +3. **임계값 정책 확정(#85)** — 정상 FPR 상한 제약이 들어가야 운영점을 정할 수 있다. + +## 7. 산출물 + +| 경로 | 내용 | +|---|---| +| `data_science/SMSModel/run_error_analysis.py` | 오탐·미탐 분석 도구 (신규) | +| `data_science/SMSModel/reports/error_analysis_baseline.json` | 특징 추가 전 기준선 | +| `data_science/SMSModel/reports/error_analysis_with_ad_features.json` | 특징 추가 후 결과 | +| `data_science/SMSModel/artifacts/stacking/v3-baseline/` | 특징 추가 전 artifact (보존) | +| `data_science/SMSModel/artifacts/stacking/v3/` | 특징 추가 후 artifact | +| `data_science/SMSModel/reports/stacking_v3-baseline/` | 특징 추가 전 평가 리포트 | +| `data_science/SMSModel/reports/stacking_v3/` | 특징 추가 후 평가 리포트 | +| `app/analysis/text/structural_features.py` | 광고 표기 특징 추가 | + +## 8. 방법론 기록 + +- 임계값은 validation에서만 선정했고 test·holdout으로 재조정하지 않았다. +- 데이터셋은 이 실험 동안 변경하지 않았다. 특징 추가만이 유일한 변경이다. +- 오탐 원인 분석 과정에서 real_holdout을 참조했다. 다만 광고 집중 현상은 test split에서도 동일하게 관측되므로(5/8), 특징 설계가 holdout에만 의존하지 않는다. +- `v2` artifact와 `#78` 리포트는 `REJECT_CURRENT_ARTIFACT` 상태로 보존했다. +- `v3`와 `v3-baseline`은 동일한 `sms_split_v3.csv`를 사용했다. 두 metadata의 `split_manifest_sha256`이 한때 달랐던 것은 split 구성 차이가 아니라 Windows 체크아웃(`core.autocrlf=true`)에서 CRLF로 변환된 사본을 해싱했기 때문이다. 줄바꿈만 정규화하면 두 값이 `459c24…`로 일치한다. `.gitattributes`에 `eol=lf`를 고정해 재발을 막고 metadata를 정정했다. split 멤버십이 동일하므로 재학습은 필요하지 않다. +- `has_ad_disclosure`에 `re.MULTILINE`을 추가하고 `has_opt_out`의 중복 분기를 정리했으나, 데이터셋 3,078행 전체에서 두 특징값 변화는 0건이라 위 수치와 artifact는 그대로 유효하다. diff --git a/tests/analysis/text/test_structural_features.py b/tests/analysis/text/test_structural_features.py index 82540e2..e3462a9 100644 --- a/tests/analysis/text/test_structural_features.py +++ b/tests/analysis/text/test_structural_features.py @@ -74,3 +74,83 @@ def test_empty_input_returns_empty_matrix() -> None: def test_rejects_non_string_input() -> None: with pytest.raises(TypeError, match="text must be a string"): extract_stacking_structural_features(123) + + +def test_detects_legal_advertising_disclosure() -> None: + """법정 광고 표기와 수신거부 안내를 특징으로 잡아내는지 검증""" + result = extract_stacking_structural_features( + "(광고)[나이키] 시즌 마감 세일 40% 할인 무료수신거부 080-123-4567" + ) + values = dict(zip(result.names, result.values, strict=True)) + + assert values["has_ad_disclosure"] == 1.0 + assert values["has_opt_out"] == 1.0 + + +def test_detects_bracket_style_advertising_disclosure() -> None: + """[광고] 형태와 공백이 섞인 표기도 인식해야 한다""" + result = extract_stacking_structural_features( + "[ 광고 ] 신규 회원 쿠폰 안내입니다. 수신 거부 080 1234 5678" + ) + values = dict(zip(result.names, result.values, strict=True)) + + assert values["has_ad_disclosure"] == 1.0 + assert values["has_opt_out"] == 1.0 + + +def test_plain_notification_has_no_advertising_marks() -> None: + """광고 표기가 없는 정상 알림은 두 특징 모두 0이어야 한다""" + result = extract_stacking_structural_features( + "[국민은행] 출금 50,000원 잔액 120,000원" + ) + values = dict(zip(result.names, result.values, strict=True)) + + assert values["has_ad_disclosure"] == 0.0 + assert values["has_opt_out"] == 0.0 + + +def test_feature_matrix_width_matches_names() -> None: + """특징을 추가해도 행렬 폭과 이름 개수가 일치해야 한다""" + matrix = extract_stacking_structural_matrix(["테스트 문자", "두 번째 문자"]) + + assert matrix.shape == (2, len(STACKING_STRUCTURAL_FEATURE_NAMES)) + + +def test_advertising_features_are_appended_last() -> None: + """열 순서 호환을 위해 새 특징은 항상 마지막에 있어야 한다""" + # 콤마 누락으로 인접 문자열이 암묵적으로 이어붙으면 개수부터 어긋난다. + assert len(STACKING_STRUCTURAL_FEATURE_NAMES) == 14 + assert STACKING_STRUCTURAL_FEATURE_NAMES[-2:] == ( + "has_ad_disclosure", + "has_opt_out", + ) + + +def test_detects_advertising_disclosure_after_web_prefix() -> None: + """`[Web발신]` 접두어 다음 줄에서 시작하는 광고 표기도 인식해야 한다""" + result = extract_stacking_structural_features( + "[Web발신]\n광고 신상품 안내입니다.\n무료수신거부 080-123-4567" + ) + values = dict(zip(result.names, result.values, strict=True)) + + assert values["has_ad_disclosure"] == 1.0 + assert values["has_opt_out"] == 1.0 + + +def test_advertising_disclosure_ignores_mid_line_mention() -> None: + """문장 중간에 언급된 '광고'는 법정 표기로 보지 않는다""" + result = extract_stacking_structural_features( + "어제 본 광고 기억나? 그거 링크 좀 보내줘." + ) + values = dict(zip(result.names, result.values, strict=True)) + + assert values["has_ad_disclosure"] == 0.0 + + +def test_opt_out_covers_free_variants() -> None: + """무료수신거부·수신 거부·무료거부 표기를 모두 같은 특징으로 잡는다""" + for text in ("무료수신거부", "수신 거부", "무료거부", "080-123-4567"): + result = extract_stacking_structural_features(text) + values = dict(zip(result.names, result.values, strict=True)) + + assert values["has_opt_out"] == 1.0, text