diff --git a/.dprint.json b/.dprint.json index bd1279fd4a..683186177e 100644 --- a/.dprint.json +++ b/.dprint.json @@ -45,6 +45,7 @@ "tests/specs/fmt", "tests/specs/lint/bom", "tests/specs/lint/default_ts", + "tests/specs/lint/minified", "tests/specs/lint/syntax_error_reporting", "tests/specs/publish/no_check_surfaces_syntax_error", "tests/specs/run/default_ts", diff --git a/cli/tools/lint/linter.rs b/cli/tools/lint/linter.rs index 5d6f845274..d321356b1a 100644 --- a/cli/tools/lint/linter.rs +++ b/cli/tools/lint/linter.rs @@ -17,12 +17,24 @@ use deno_lint::linter::Linter as DenoLintLinter; use deno_lint::linter::LinterOptions; use deno_path_util::fs::atomic_write_file_with_retries; +use super::minified_file; use super::rules::FileOrPackageLintRule; use super::rules::PackageLintRule; use super::ConfiguredRules; use crate::sys::CliSys; use crate::util::fs::specifier_from_file_path; +pub enum LintResult { + /// File was linted and optionally produced diagnostics + Linted { + parsed_source: ParsedSource, + diagnostics: Vec, + }, + /// File was not parsed and linted because, eg. it might have + /// been a minified file. + Skipped { reason: String }, +} + pub struct CliLinterOptions { pub configured_rules: ConfiguredRules, pub fix: bool, @@ -95,8 +107,15 @@ impl CliLinter { file_path: &Path, source_code: String, ext: Option<&str>, - ) -> Result<(ParsedSource, Vec), AnyError> { + ) -> Result { let specifier = specifier_from_file_path(file_path)?; + + if minified_file::is_likely_minified(&source_code) { + return Ok(LintResult::Skipped { + reason: "The file is minified".to_string(), + }); + } + let media_type = if let Some(ext) = ext { MediaType::from_str(&format!("placeholder.{ext}")) } else if file_path.extension().is_none() { @@ -108,7 +127,7 @@ impl CliLinter { if self.fix { self.lint_file_and_fix(&specifier, media_type, source_code, file_path) } else { - self + let (parsed_source, diagnostics) = self .linter .lint_file(LintFileOptions { specifier, @@ -116,7 +135,11 @@ impl CliLinter { source_code, config: self.deno_lint_config.clone(), }) - .map_err(AnyError::from) + .map_err(AnyError::from)?; + Ok(LintResult::Linted { + parsed_source, + diagnostics, + }) } } @@ -126,7 +149,7 @@ impl CliLinter { media_type: MediaType, source_code: String, file_path: &Path, - ) -> Result<(ParsedSource, Vec), deno_core::anyhow::Error> { + ) -> Result { // initial lint let (source, diagnostics) = self.linter.lint_file(LintFileOptions { specifier: specifier.clone(), @@ -184,7 +207,10 @@ impl CliLinter { .context("Failed writing fix to file.")?; } - Ok((source, diagnostics)) + Ok(LintResult::Linted { + parsed_source: source, + diagnostics, + }) } } diff --git a/cli/tools/lint/minified_file.rs b/cli/tools/lint/minified_file.rs new file mode 100644 index 0000000000..57047a4666 --- /dev/null +++ b/cli/tools/lint/minified_file.rs @@ -0,0 +1,180 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +#[derive(Debug)] +pub struct FileMetrics { + long_lines_count: usize, + total_lines: usize, + whitespace_ratio: f64, + has_license_comment: bool, +} + +impl FileMetrics { + #[inline] + pub fn is_likely_minified(&self) -> bool { + let long_lines_ratio = + self.long_lines_count as f64 / self.total_lines as f64; + + (long_lines_ratio >= 0.2 || self.whitespace_ratio < 0.05) + && !(self.has_license_comment && self.total_lines < 3) + } +} + +/// Analyze the content and tell if the file is most likely a minified file or not. +pub fn is_likely_minified(content: &str) -> bool { + const LONG_LINE_LEN: usize = 250; + let mut total_lines = 0; + let mut long_lines_count = 0; + let mut whitespace_count = 0; + let mut total_chars = 0; + let mut has_license = false; + let mut in_multiline_comment = false; + + // If total len of a file is shorter than the "long line" length, don't bother analyzing + // and consider non-minified. + if content.len() < LONG_LINE_LEN { + return false; + } + + let mut str_ref = content; + + // Process the content character by character to avoid line allocations + let mut chars = content.chars().enumerate().peekable(); + while let Some((idx, c)) = chars.next() { + total_chars += 1; + + if c.is_whitespace() { + whitespace_count += 1; + } + + str_ref = &content[..idx]; + + // Check for end of line or end of content + if c == '\n' || chars.peek().is_none() { + total_lines += 1; + let trimmed = str_ref.trim(); + + // Check for license/copyright only if we haven't found one yet + if !has_license && !trimmed.is_empty() { + // Avoid allocating a new string for case comparison + has_license = trimmed.chars().any(|c| c.is_ascii_alphabetic()) + && (trimmed.contains("license") + || trimmed.contains("LICENSE") + || trimmed.contains("copyright") + || trimmed.contains("COPYRIGHT") + || trimmed.contains("(c)") + || trimmed.contains("(C)")); + } + + // Handle comments without allocating new strings + if trimmed.starts_with("/*") { + in_multiline_comment = true; + } + if trimmed.ends_with("*/") { + in_multiline_comment = false; + str_ref = ""; + continue; + } + if in_multiline_comment || trimmed.starts_with("//") { + str_ref = ""; + continue; + } + + // Check line length + if str_ref.len() > LONG_LINE_LEN { + long_lines_count += 1; + } + + str_ref = ""; + } + } + + // Handle case where file doesn't end with newline + if !str_ref.is_empty() { + total_lines += 1; + } + + let whitespace_ratio = if total_chars > 0 { + whitespace_count as f64 / total_chars as f64 + } else { + 0.0 + }; + + let metrics = FileMetrics { + long_lines_count, + total_lines, + whitespace_ratio, + has_license_comment: has_license, + }; + + metrics.is_likely_minified() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_normal_js() { + let content = r#" +function hello() { + // This is a normal comment + console.log("Hello, world!"); +} + +// Another comment +const x = 42; + +/* Multi-line + comment */ +"#; + assert!(!is_likely_minified(content)); + } + + #[test] + fn empty_file() { + assert!(!is_likely_minified("")); + } + + #[test] + fn test_minified_file_col_length() { + let content = + "const LOREM_IPSUM = `Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.`"; + assert!(is_likely_minified(content)); + } + + #[test] + fn test_minified_js() { + let content = "function hello(){console.log(\"Hello, world!\")}const x=42;function veryLongFunction(){return\"This is a very long line that exceeds 250 characters and contains lots of code and stuff and more code and even more stuff until we definitely exceed the limit we set for considering a line to be very long in our minification detection algorithm\"}"; + assert!(is_likely_minified(content)); + } + + #[test] + fn test_minified_file_whitespace() { + let content = + "function f(a,b){return a.concat(b)}var x=function(n){return n+1};"; + assert!(!is_likely_minified(content)); + } + + #[test] + fn test_license_only() { + let content = r#"/* +* Copyright (c) 2024 Example Corp. +* Licensed under MIT License +*/ +"#; + assert!(!is_likely_minified(content)); + } + + #[test] + fn test_normal_file() { + let content = r#" +function concatenateArrays(array1, array2) { + return array1.concat(array2); +} + +const incrementNumber = function(number) { + return number + 1; +};"#; + assert!(!is_likely_minified(content)); + } +} diff --git a/cli/tools/lint/mod.rs b/cli/tools/lint/mod.rs index 36ba85f613..43c98366b4 100644 --- a/cli/tools/lint/mod.rs +++ b/cli/tools/lint/mod.rs @@ -12,7 +12,6 @@ use std::rc::Rc; use std::sync::Arc; use deno_ast::ModuleSpecifier; -use deno_ast::ParsedSource; use deno_config::deno_json::LintRulesConfig; use deno_config::glob::FileCollector; use deno_config::glob::FilePatterns; @@ -26,7 +25,6 @@ use deno_core::serde_json; use deno_core::unsync::future::LocalFutureExt; use deno_core::unsync::future::SharedLocal; use deno_graph::ModuleGraph; -use deno_lint::diagnostic::LintDiagnostic; use deno_lint::linter::LintConfig as DenoLintConfig; use log::debug; use reporters::create_reporter; @@ -55,6 +53,7 @@ use crate::util::sync::AtomicFlag; mod ast_buffer; mod linter; +mod minified_file; mod reporters; mod rules; @@ -62,6 +61,7 @@ mod rules; pub use ast_buffer::serialize_ast_to_buffer; pub use linter::CliLinter; pub use linter::CliLinterOptions; +use linter::LintResult; pub use rules::collect_no_slow_type_diagnostics; pub use rules::ConfiguredRules; pub use rules::LintRuleProvider; @@ -316,7 +316,6 @@ impl WorkspaceLinter { let fut = async move { let operation = move |file_path: PathBuf| { let file_text = deno_ast::strip_bom(fs::read_to_string(&file_path)?); - // don't bother rechecking this file if it didn't have any diagnostics before if let Some(incremental_cache) = &maybe_incremental_cache_ { if incremental_cache.is_file_same(&file_path, &file_text) { @@ -329,14 +328,18 @@ impl WorkspaceLinter { file_text, cli_options.ext_flag().as_deref(), ); - if let Ok((file_source, file_diagnostics)) = &r { + if let Ok(LintResult::Linted { + parsed_source, + diagnostics, + }) = &r + { if let Some(incremental_cache) = &maybe_incremental_cache_ { - if file_diagnostics.is_empty() { + if diagnostics.is_empty() { // update the incremental cache if there were no diagnostics incremental_cache.update_file( &file_path, // ensure the returned text is used here as it may have been modified via --fix - file_source.text(), + parsed_source.text(), ) } } @@ -552,34 +555,43 @@ fn lint_stdin( fn handle_lint_result( file_path: &str, - result: Result<(ParsedSource, Vec), AnyError>, + result: Result, reporter_lock: Arc>>, ) -> bool { let mut reporter = reporter_lock.lock(); match result { - Ok((source, mut file_diagnostics)) => { - if !source.diagnostics().is_empty() { - for parse_diagnostic in source.diagnostics() { - log::warn!("{}: {}", colors::yellow("warn"), parse_diagnostic); - } - } - file_diagnostics.sort_by(|a, b| match a.specifier.cmp(&b.specifier) { - std::cmp::Ordering::Equal => { - let a_start = a.range.as_ref().map(|r| r.range.start); - let b_start = b.range.as_ref().map(|r| r.range.start); - match a_start.cmp(&b_start) { - std::cmp::Ordering::Equal => a.details.code.cmp(&b.details.code), - other => other, + Ok(lint_result) => match lint_result { + LintResult::Linted { + parsed_source, + mut diagnostics, + } => { + if !parsed_source.diagnostics().is_empty() { + for parse_diagnostic in parsed_source.diagnostics() { + log::warn!("{}: {}", colors::yellow("warn"), parse_diagnostic); } } - file_order => file_order, - }); - for d in &file_diagnostics { - reporter.visit_diagnostic(d); + diagnostics.sort_by(|a, b| match a.specifier.cmp(&b.specifier) { + std::cmp::Ordering::Equal => { + let a_start = a.range.as_ref().map(|r| r.range.start); + let b_start = b.range.as_ref().map(|r| r.range.start); + match a_start.cmp(&b_start) { + std::cmp::Ordering::Equal => a.details.code.cmp(&b.details.code), + other => other, + } + } + file_order => file_order, + }); + for d in &diagnostics { + reporter.visit_diagnostic(d); + } + diagnostics.is_empty() } - file_diagnostics.is_empty() - } + LintResult::Skipped { reason } => { + reporter.visit_skipped(file_path, &reason); + true + } + }, Err(err) => { reporter.visit_error(file_path, &err); false @@ -593,6 +605,12 @@ struct LintError { message: String, } +#[derive(Serialize)] +struct LintSkipped { + file_path: String, + message: String, +} + #[cfg(test)] mod tests { use pretty_assertions::assert_eq; diff --git a/cli/tools/lint/reporters.rs b/cli/tools/lint/reporters.rs index 24e04e840f..95549345c4 100644 --- a/cli/tools/lint/reporters.rs +++ b/cli/tools/lint/reporters.rs @@ -9,6 +9,7 @@ use log::info; use serde::Serialize; use super::LintError; +use super::LintSkipped; use crate::args::LintReporterKind; const JSON_SCHEMA_VERSION: u8 = 1; @@ -23,6 +24,7 @@ pub fn create_reporter(kind: LintReporterKind) -> Box { pub trait LintReporter { fn visit_diagnostic(&mut self, d: &LintDiagnostic); + fn visit_skipped(&mut self, file_path: &str, reason: &str); fn visit_error(&mut self, file_path: &str, err: &AnyError); fn close(&mut self, check_count: usize); } @@ -51,6 +53,11 @@ impl LintReporter for PrettyLintReporter { log::error!("{}\n", d.display()); } + fn visit_skipped(&mut self, file_path: &str, reason: &str) { + log::info!("File was skipped: {file_path}"); + log::info!(" {reason}"); + } + fn visit_error(&mut self, file_path: &str, err: &AnyError) { log::error!("Error linting: {file_path}"); log::error!(" {err}"); @@ -112,6 +119,11 @@ impl LintReporter for CompactLintReporter { } } + fn visit_skipped(&mut self, file_path: &str, reason: &str) { + log::info!("File was skipped: {file_path}"); + log::info!(" {reason}"); + } + fn visit_error(&mut self, file_path: &str, err: &AnyError) { log::error!("Error linting: {file_path}"); log::error!(" {err}"); @@ -173,6 +185,7 @@ struct JsonLintDiagnostic { struct JsonLintReporter { version: u8, diagnostics: Vec, + skipped: Vec, errors: Vec, checked_files: Vec, } @@ -182,6 +195,7 @@ impl JsonLintReporter { JsonLintReporter { version: JSON_SCHEMA_VERSION, diagnostics: Vec::new(), + skipped: Vec::new(), errors: Vec::new(), checked_files: Vec::new(), } @@ -223,6 +237,13 @@ impl LintReporter for JsonLintReporter { } } + fn visit_skipped(&mut self, file_path: &str, reason: &str) { + self.skipped.push(LintSkipped { + file_path: file_path.to_string(), + message: reason.to_string(), + }); + } + fn visit_error(&mut self, file_path: &str, err: &AnyError) { self.errors.push(LintError { file_path: file_path.to_string(), diff --git a/tests/specs/lint/json/expected_json.out b/tests/specs/lint/json/expected_json.out index 242c47cf56..7db69f6f64 100644 --- a/tests/specs/lint/json/expected_json.out +++ b/tests/specs/lint/json/expected_json.out @@ -56,6 +56,7 @@ "hint": [WILDCARD] } ], + "skipped": [], "errors": [ { "file_path": "[WILDCARD]malformed.js", diff --git a/tests/specs/lint/minified/__test__.jsonc b/tests/specs/lint/minified/__test__.jsonc new file mode 100644 index 0000000000..6d71840d5a --- /dev/null +++ b/tests/specs/lint/minified/__test__.jsonc @@ -0,0 +1,5 @@ +{ + "args": ["lint", "minified.js"], + "output": "minified.out", + "exitCode": 0 +} diff --git a/tests/specs/lint/minified/minified.js b/tests/specs/lint/minified/minified.js new file mode 100644 index 0000000000..5db24c1509 --- /dev/null +++ b/tests/specs/lint/minified/minified.js @@ -0,0 +1,5 @@ +/* +* Copyright (c) 2024 Example Corp. +* Licensed under MIT License +*/ +function f(a,b){return a.concat(b)}var x=function(n){return n+1};function g(a,b){return a.concat(b)}var x=function(n){return n+1};function h(a,b){return a.concat(b)}var x=function(n){return n+1};function i(a,b){return a.concat(b)}var x=function(n){return n+1};function j(a,b){return a.concat(b)}var x=function(n){return n+1}; \ No newline at end of file diff --git a/tests/specs/lint/minified/minified.out b/tests/specs/lint/minified/minified.out new file mode 100644 index 0000000000..b3fb1a6794 --- /dev/null +++ b/tests/specs/lint/minified/minified.out @@ -0,0 +1,3 @@ +File was skipped: [WILDCARD]minified.js + The file is minified +Checked 1 file diff --git a/tests/specs/lint/stdin_json/expected_from_stdin_json.out b/tests/specs/lint/stdin_json/expected_from_stdin_json.out index 8db157bccb..094cd85101 100644 --- a/tests/specs/lint/stdin_json/expected_from_stdin_json.out +++ b/tests/specs/lint/stdin_json/expected_from_stdin_json.out @@ -20,6 +20,7 @@ "hint": [WILDCARD] } ], + "skipped": [], "errors": [], "checked_files": [ "[WILDCARD]main.ts" diff --git a/tests/specs/lint/with_report_config_override/with_report_config_override.out b/tests/specs/lint/with_report_config_override/with_report_config_override.out index fb873f187e..1e62a8f693 100644 --- a/tests/specs/lint/with_report_config_override/with_report_config_override.out +++ b/tests/specs/lint/with_report_config_override/with_report_config_override.out @@ -38,6 +38,7 @@ "hint": "If this is intentional, prefix it with an underscore like `_add`" } ], + "skipped": [], "errors": [], "checked_files": [ "[WILDCARD]a.ts"