Блог

Я понятия не имею, что здесь происходит, связи быть не должно. В любом случае, это не то, что проявляется конкретно.

После переноса buildscript основного проекта spring-boot с groovy на kotlin-DSL некоторые модульные тесты выдают исключение NullPointerException. Все модульные тесты имеют дело с методами, которые имеют ручное управление транзакциями с использованием TransactionTemplate. Этого не происходило до перехода, и к настоящему времени я сравнил старый и новый buildscript дюжину раз и вполне уверен, что ничего не забыл.

Давайте начнем с сравнения двух версий сценариев сборки.

Старый groovy:

 buildscript {
    ext {
        kotlinVersion = '1.4.10'
        springBootVersion = '2.3.3.RELEASE'
        awsSdkVersion = '1.11.381'
        springfoxVersion = '2.9.2'
        kotlintestVersion = '3.1.7'
    }
    repositories {
        mavenLocal()
        jcenter()
        mavenCentral()
    }
    dependencies {
        classpath "org.springframework.boot:spring-boot-gradle-plugin:$springBootVersion"
        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion"
        classpath "org.jetbrains.kotlin:kotlin-allopen:$kotlinVersion"
        classpath "org.jetbrains.kotlin:kotlin-noarg:$kotlinVersion"

        // liquibase stuff
        classpath 'mysql:mysql-connector-java:5.1.44'
        classpath 'org.yaml:snakeyaml:1.19'
        classpath('org.liquibase:liquibase-gradle-plugin:1.2.3') {
            exclude group: "org.liquibase", module: "liquibase-core"
        }
        classpath "org.liquibase:liquibase-core:3.5.3" // should be in sync with the version included in spring boot
    }
}

plugins {
    id 'net.researchgate.release' version '2.6.0'
}

apply plugin: 'kotlin'
apply plugin: 'kotlin-spring'
apply plugin: 'kotlin-jpa'
apply plugin: 'org.springframework.boot'
apply plugin: 'io.spring.dependency-management'
apply plugin: 'jacoco'
apply plugin: 'org.liquibase.gradle'
apply plugin: 'idea'

group = 'webcam.yellow.service'
sourceCompatibility = 1.8

compileKotlin {
    kotlinOptions.jvmTarget = "1.8"
}
compileTestKotlin {
    kotlinOptions.jvmTarget = "1.8"
}

jacocoTestReport {
    reports {
        xml.enabled = true
        html.enabled = true
    }
}

check.dependsOn jacocoTestReport

repositories {
    mavenLocal()
    mavenCentral()
    jcenter()
    maven { url = "http://clojars.org/repo/" }
    maven {
        credentials {
            username "${mavenUser}"
            password "${mavenPassword}"
        }
        url = "https://artifactory.yellow.webcam/artifactory/releases"
    }

    maven {
        credentials {
            username "${mavenUser}"
            password "${mavenPassword}"
        }
        url = "https://artifactory.yellow.webcam/artifactory/snapshots"
    }

}

idea {
    module {
        // if you hate browsing Javadoc
        downloadJavadoc = false
        // and love reading sources :)
        downloadSources = true
    }
}

dependencies {
    // avisec libraries
    compile "webcam.yellow.api:webcam-api:3.2.0"
    compile "webcam.yellow.authentication:messaging-authentication:1.5"

    // Spring Boot
    compile 'org.springframework.boot:spring-boot-starter-actuator'
    compile 'org.springframework.boot:spring-boot-starter-data-jpa'
    compile 'org.springframework.boot:spring-boot-starter-web'
    compile 'org.springframework.boot:spring-boot-starter-security'
    compile 'org.springframework.boot:spring-boot-starter-activemq'
    compile 'org.springframework.boot:spring-boot-starter-validation'

    compile 'io.micrometer:micrometer-registry-influx'

    compile 'com.fasterxml.jackson.module:jackson-module-kotlin'
    compile 'com.fasterxml.jackson.datatype:jackson-datatype-joda'

    // Joda
    compile 'joda-time:joda-time'
    compile 'org.jadira.usertype:usertype.core:6.0.1.GA'

    // Libraries
    compile group: 'org.apache.commons', name: 'commons-text', version: '1.1'
    compile group: 'org.apache.commons', name: 'commons-csv', version: '1.8'

    // Kotlin
    compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${kotlinVersion}"
    compile "org.jetbrains.kotlin:kotlin-reflect:${kotlinVersion}"

    // Mail
    compile group: 'org.springframework.boot', name: 'spring-boot-starter-mail', version: '2.2.4.RELEASE'
    compile group: 'org.apache.commons', name: 'commons-email', version: '1.5'
    compile group: 'javax.mail', name: 'mail', version: '1.5.0-b01'

    // AWS
    compile "com.amazonaws:aws-java-sdk-s3:$awsSdkVersion"
    compile group: 'com.amazonaws', name: 'aws-java-sdk-sqs', version: '1.11.558'
    compile group: 'software.amazon.awssdk', name: 's3', version: '2.10.61'
    compile group: 'com.amazonaws', name: 'aws-java-sdk-ec2', version: '1.11.740'

    // Support old pw hash lib
    compile group: 'buddy', name: 'buddy-hashers', version: '1.3.0'

    // Hazelcast
    compile group: 'com.hazelcast', name: 'hazelcast-spring'
    compile group: 'com.hazelcast', name: 'hazelcast-aws', version: '2.4'

    // Swagger
    compile group: 'io.springfox', name: 'springfox-swagger2', version: springfoxVersion
    compile group: 'io.springfox', name: 'springfox-swagger-ui', version: springfoxVersion

    // Database
    runtime 'mysql:mysql-connector-java'
    runtime 'org.liquibase:liquibase-core'
    compile 'org.influxdb:influxdb-java:2.13'

    // HTML Generation
    compile group: 'org.jetbrains.kotlinx', name: 'kotlinx-html-jvm', version: '0.6.10'
    //  ActiveMQ
    compile group: 'org.messaginghub', name: 'pooled-jms'

    // Development
    compile "org.springframework.boot:spring-boot-devtools"

    // Testing
    testCompile 'org.springframework.boot:spring-boot-starter-test'
    testCompile 'org.springframework.security:spring-security-test'
    testCompile 'com.h2database:h2'
    testCompile 'io.kotlintest:kotlintest:2.0.7'
    testCompile 'com.nhaarman:mockito-kotlin:1.6.0'


    // GSON
    compile group: 'com.google.code.gson', name: 'gson', version: '2.8.6'
}

liquibase {
    activities {
        main {
            changeLogFile '/db/changelog/db.changelog-master.yaml'
            url 'jdbc:mysql://localhost:3307/yellow'
            username 'user'
            password 'secret'
            classpath 'src/main/resources'
        }
    }
}


springBoot {
    buildInfo()
}

task ebextensions(type: Exec) {
    executable "sh"
    args "-c", "jar uf build/libs/webcam-service*.jar .ebextensions"
}

bootJar.finalizedBy ebextensions

afterReleaseBuild.dependsOn bootJar

test {
    minHeapSize = "1024m"
    maxHeapSize = "1024m"
    jvmArgs = ["-Xloggc:build/gclog-%p.log", "-XX: PrintGCDetails"]
}
  

И новый kotlin:

 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile

buildscript {
    dependencies {
        classpath("org.jfrog.buildinfo:build-info-extractor-gradle:4.9.6")
        classpath("org.jetbrains.kotlin:kotlin-allopen:1.4.10")
        // liquibase stuff
        classpath("mysql:mysql-connector-java:5.1.44")
        classpath("org.yaml:snakeyaml:1.19")
        classpath("org.liquibase:liquibase-gradle-plugin:1.2.3") {
            exclude("org.liquibase", "liquibase-core")
        }
        classpath("org.liquibase:liquibase-core:3.8.9") // should be in sync with the version included in spring boot
    }
}

val awsSdkVersion = "1.11.740"
val springfoxVersion = "2.9.2"


plugins {
    id("org.springframework.boot") version "2.3.3.RELEASE"
    id("io.spring.dependency-management") version "1.0.9.RELEASE"
    id("net.researchgate.release") version "2.6.0"
    kotlin("jvm") version "1.4.10"
    kotlin("plugin.spring") version "1.4.10"
    `maven-publish`
    id("com.jfrog.artifactory") version "4.14.1"
    id("org.liquibase.gradle") version "2.0.4"
    id("org.jetbrains.kotlin.plugin.noarg") version "1.4.10"
    id("org.jetbrains.kotlin.plugin.jpa") version "1.4.10"
}

group = "webcam.yellow.service"
java.sourceCompatibility = JavaVersion.VERSION_1_8

val mavenUser: String by project
val mavenPassword: String by project
val artifactoryRepository = System.getenv("ARTIFACTORY_REPO") ?: "snapshots"


repositories {
    mavenCentral()
    jcenter()
    maven {
        credentials {
            username = mavenUser
            password = mavenPassword
        }
        url = uri("https://artifactory.yellow.webcam/artifactory/releases")
    }

    maven {
        credentials {
            username = mavenUser
            password = mavenPassword
        }
        url = uri("https://artifactory.yellow.webcam/artifactory/snapshots")
    }
}

dependencies {

    // Spring Boot
    implementation("org.springframework.boot:spring-boot-starter-web")
    implementation("org.springframework.boot:spring-boot-starter-activemq")
    implementation("org.springframework.boot:spring-boot-starter-actuator")
    implementation("org.springframework.boot:spring-boot-starter-validation")
    implementation("org.springframework.boot:spring-boot-starter-security")
    implementation("org.springframework.boot:spring-boot-starter-data-jpa")
    implementation("org.springframework.boot:spring-boot-starter-mail")


    implementation("io.micrometer:micrometer-registry-influx")

    // Kotlin
    implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8")
    implementation("org.jetbrains.kotlin:kotlin-reflect")
    implementation("org.jetbrains.kotlinx:kotlinx-html-jvm:0.6.8")

    // Yellow Dependencies
    implementation("webcam.yellow.api:webcam-api:3.2.0")
    implementation("webcam.yellow.authentication:messaging-authentication:1.5")

    // AWS
    implementation("com.amazonaws:aws-java-sdk-s3:$awsSdkVersion")
    implementation("software.amazon.awssdk:s3:2.10.61") // does not yet incorporate all functionality of 1.11.X
    implementation("com.amazonaws:aws-java-sdk-sqs:$awsSdkVersion")
    implementation("com.amazonaws:aws-java-sdk-ec2:$awsSdkVersion")

    // Support old pw hash lib
    implementation("buddy:buddy-hashers:1.3.0")

    // Jackson
    implementation("com.fasterxml.jackson.module:jackson-module-kotlin")
    implementation("com.fasterxml.jackson.datatype:jackson-datatype-joda")

    // Hazelcast
    implementation("com.hazelcast:hazelcast-spring")
    implementation("com.hazelcast:hazelcast-aws:2.4")

    // Other Libs
    implementation("org.apache.commons:commons-text:1.1")
    implementation("org.apache.commons:commons-csv:1.8")

    implementation("org.messaginghub:pooled-jms")
    implementation("org.influxdb:influxdb-java:2.13")
    implementation("org.jetbrains.kotlinx:kotlinx-html-jvm:0.6.10")
    implementation("org.springframework.boot:spring-boot-devtools")
    implementation("com.google.code.gson:gson:2.8.6")
    implementation("joda-time:joda-time")
    implementation("org.jadira.usertype:usertype.core:6.0.1.GA")


    // Swagger
    implementation("io.springfox:springfox-swagger2:$springfoxVersion")
    implementation("io.springfox:springfox-swagger-ui:$springfoxVersion")

    // Database
    runtimeOnly("mysql:mysql-connector-java")
    runtimeOnly("org.liquibase:liquibase-core")
    implementation("org.influxdb:influxdb-java:2.13")


    // Test
    testImplementation("org.springframework.boot:spring-boot-starter-test")
    testImplementation("org.springframework.security:spring-security-test")
    testImplementation("com.h2database:h2")
    testImplementation("com.nhaarman:mockito-kotlin:1.6.0")
    testImplementation("io.kotlintest:kotlintest:2.0.7")

}

tasks.withType<Test> {
    useJUnitPlatform()
}

springBoot {
    buildInfo()
}

liquibase {
    activities.register("main") {
        arguments = mapOf(
            "changeLogFile" to "/db/changelog/db.changelog-master.yaml",
            "url" to "jdbc:mysql://localhost:3307/yellow",
            "username" to "user",
            "password" to "secret",
            "classpath" to "src/main/resources"
        )
    }
}


tasks {
    withType<KotlinCompile> {
        kotlinOptions {
            freeCompilerArgs = listOf("-Xjsr305=strict")
            jvmTarget = "1.8"
        }
    }

    bootJar {
        from("./.ebextensions") { into(".ebextensions") }
        launchScript()
    }

    afterReleaseBuild {
        dependsOn(bootJar)
    }

    test {
        minHeapSize = "1024m"
        maxHeapSize = "1024m"
        jvmArgs = listOf("-Xloggc:build/gclog-%p.log", "-XX: PrintGCDetails")
    }
}
  

Некоторые вещи требовали другой обработки, например, перемещение некоторых плагинов из classpath в раздел плагинов, расширения .ebsextensions обрабатываются по-другому, и Jacoco там больше нет, потому что мы никогда не использовали эти отчеты в jenkins. В целом, довольно нормально. Все работает, за исключением одной вещи в модульных тестах.

Here’s a class initialisation and one of the tested methods:

 @Service
class ImageService(private val imageRepository: ImageRepository,
                   private val imageSetRepository: ImageSetRepository,
                   private val permissionService: PermissionService,
                   private val s3Service: S3Service,
                   private val entityManager: EntityManager,
                   private val panoFeedRepository: PanoFeedRepository,
                   private val panoImageRepository: PanoImageRepository,
                   private val jobFacade: JobFacade,
                   private val jdbcTemplate: NamedParameterJdbcTemplate,
                   transactionManager: PlatformTransactionManager,
                   @Value("${service.cleanup.enabled}")
                   private val cleanupEnabled: Boolean) {


    private val readTransaction = org.springframework.transaction.support.TransactionTemplate(
            transactionManager,
            DefaultTransactionDefinition().apply {
                isReadOnly = true
                propagationBehavior = TransactionDefinition.PROPAGATION_REQUIRES_NEW
            })
    private val writeTransaction = org.springframework.transaction.support.TransactionTemplate(
            transactionManager,
            DefaultTransactionDefinition().apply {
                isReadOnly = false
                propagationBehavior = TransactionDefinition.PROPAGATION_REQUIRES_NEW
            })


    /**
     * Deletes all passed images from the database while removing them from S3 asynchronously.
     * Does not check any permissions.
     * This opens its own write transaction, which could take a long time. If individual transactions take too long,
     * invoke this method multiple times with smaller chunks to avoid issues with table locking.
     * Since this is potentially long-running, commit preceding transactions beforehand.
     */
    @Transactional(propagation = Propagation.NEVER)
    internal fun delete(images: List<ImageIdAndImageKeyAndImagePreviewKey>) {
        images.chunked(500).forEach { chunk ->
            log.debug("Deleting s3 files of {} images", chunk.size)
            jobFacade.executeAsync {
                val keysToDelete = chunk
                        .flatMap { listOf(it.getImageKey(), it.getImagePreviewKey()) }
                        .asSequence()
                        .filterNotNull()
                        .filter { !it.isBlank() }
                        .toList()


                s3Service.deleteImages(keysToDelete)
                log.trace("deleted {} s3 files of {} images", keysToDelete.size, chunk.size)
            }
            log.debug("Deleting {} images from database", chunk.size)
            writeTransaction.execute {
                imageRepository.deleteAllByIdIn(chunk.map { it.getId()!! })
            }   // <---- Exception happens in here
            log.trace("Deleted {} images from database", chunk.size)
        }
    }
}
  

И вот тестовая настройка и тест, который проверяет метод tat:

 class ImageServiceTest {

    private val imageRepository: ImageRepository = mock()
    private val imageSetRepository: ImageSetRepository = mock()
    private val permissionService: PermissionService = mock()
    private val s3Service: S3Service = mock()
    private val hazelCastInstance: HazelcastInstance = mock()
    private val panoFeedRepository: PanoFeedRepository = mock()
    private val panoImageRepository: PanoImageRepository = mock()
    private val jdbcTemplate: NamedParameterJdbcTemplate = mock()
    private val transactionManager: PlatformTransactionManager = mock()
    private val jobFacade: JobFacade

    private val sut: ImageService

    init {
        val imap: IMap<String, String> = mock()
        `when`(hazelCastInstance.getMap<String, String>(any())).thenReturn(imap)
        jobFacade = JobFacade(hazelCastInstance)
        sut = ImageService(
                imageRepository, imageSetRepository, permissionService, s3Service, mock(),
                panoFeedRepository, panoImageRepository, jobFacade, jdbcTemplate, transactionManager, false)
    }

    @Test
    fun `deleting multiple images invokes S3Service in chunks and deletes images from DB`() {
        val user = TestModelFactory.userWithOneCameraAndTablePermission()
        val table = user.tablePermissions.first().table!!
        val images = (1..800).map { TestModelFactory.image(table = table) }

        sut.delete(images.map { ImageIdentifiers(it.id, it.imageKey, it.imagePreviewKey) })
        Thread.sleep(2000) //leave enough time for S3 uploads to finish

        val idChunks = images.map { it.id }.chunked(500)
        verify(imageRepository, times(1)).deleteAllByIdIn(eq(idChunks[0]))
        verify(imageRepository, times(1)).deleteAllByIdIn(eq(idChunks[1]))
        verify(s3Service, times(2)).deleteImages(any())
    }
  

И, наконец, ошибка:

 java.lang.NullPointerException: Parameter specified as non-null is null: method webcam.yellow.service.service.ImageService$delete$$inlined$forEach$lambda$2.doInTransaction, parameter it
  

В настоящее время я изучаю несколько вопросов о том, что может пойти не так, включая издевательский TransactionManager и не издевательские TransactionTemplates… Но действительно неприятно то, что этого просто не произошло до переноса сценария сборки, и больше ничего не изменилось.

Оказывается, TransactionTemplate (который не издевается) вызвал метод в макете PlatformTransactionManager, который вернул null (очевидно), и это вызвало исключение NullPointerException.

Это было исправлено относительно легко, настроив макет экземпляра на возврат значения вместо:

 given(transactionManager.getTransaction(any())).willReturn(mock())
  

Чего я не знаю, так это почему это когда-либо работало раньше. Эта проблема всегда должна была быть, но ее не было… Только когда я перенес файл сборки, он показал свою головку, что на самом деле не имеет особого смысла.