Category: Research and development

Bash script setup basically VM

No password required with sudo

joos ALL=(ALL) NOPASSWD:ALL

Extend LVM

sudo lvextend -l +100%FREE /dev/ubuntu-vg/ubuntu-lv
sudo resize2fs /dev/mapper/ubuntu--vg-ubuntu--lv

Setup timezone

sudo timedatectl set-timezone Asia/Ho_Chi_Minh
sudo timedatectl set-ntp on

Docker

sudo apt-get update
sudo apt-get install ca-certificates curl gnupg
sudo install -m 0755 -d /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
sudo chmod a+r /etc/apt/keyrings/docker.gpg

# Add the repository to Apt sources:
echo \
  "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
  "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update

sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
sudo groupadd docker
sudo usermod -aG docker $USER
newgrp docker

Setup PHP

#!/bin/bash

echo 'Install requirements'

sudo apt install software-properties-common
sudo add-apt-repository ppa:ondrej/php
sudo apt update && sudo apt upgrade -y

phpVersions=('8.3')
phpExtensions=('dev' 'cli' 'mbstring' 'curl' 'intl' 'mbstring' 'xml' 'xmlrpc' 'xsl' 'yaml' 'zip' 'imagick' 'gd' 'opcache' 'memcache' 'memcached' 'mysql' 'sqlite3' 'ldap' 'bcmath' 'fpm')
phpPecls=('mongodb' 'redis' 'pcov' 'apcu')

for phpVersion in "${phpVersions[@]}"
do
  echo "Install PHP ${phpVersion} extensions"
  extensions=$(printf "php${phpVersion}-%s " "${phpExtensions[@]}")
  sudo apt install -y $extensions

  echo "Install PHP ${phpVersion} pecl extensions"
  for phpPecl in "${phpPecls[@]}"
  do
    sudo pecl -d php_suffix="${phpVersion}" install "${phpPecl}"
    sudo pecl uninstall -r "${phpPecl}"    
    sudo bash -c 'echo "extension='${phpPecl}'.so" >> /etc/php/'${phpVersion}'/cli/php.ini'
    sudo bash -c 'echo "extension='${phpPecl}'.so" >> /etc/php/'${phpVersion}'/fpm/php.ini'
  done
done

Github action – Reusable

# https://docs.github.com/en/actions/learn-github-actions/understanding-github-actions
name: XCrawler - Build & Tests

# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idruns-on
on:
  # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows
  pull_request:
    branches: [ develop ]
    types: [ opened, synchronize ]

jobs:

  sonar_cloud:
    name: SonarCloud
    uses: ./.github/workflows/sonar_cloud.yml
    secrets: inherit

  security_check:
    name: Security check
    uses: ./.github/workflows/security_check.yml
    secrets: inherit

  code_standards:
    name: Code standards check
    needs: [ sonar_cloud, security_check ]
    strategy:
      matrix:
        lint: [ phpstan, phpmd, phpcs ]
    uses: ./.github/workflows/code_standards.yml
    with:
      lint: ${{ matrix.lint }}
    secrets: inherit

  tests:
    name: Execute UnitTest
    needs: [ code_standards ]
    strategy:
      matrix:
        test: [ Client, JAV, Flickr, Core ]
    uses: ./.github/workflows/unittest.yml
    with:
      test: ${{ matrix.test }}
    secrets: inherit

  finalizing_build:
    name: "All Parallel Tests passed"
    needs: [ tests ]
    uses: ./.github/workflows/codecov.yml
    secrets: inherit
Continue reading Github action – Reusable

Github action – Slack notification

Here is good one package to use

https://github.com/slackapi/slack-github-action

And here is my implement

      - name: Slack Notification
        id: slack
        uses: slackapi/[email protected]
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
        with:
          # Slack channel id, channel name, or user id to post message.
          # See also: https://api.slack.com/methods/chat.postMessage#channels
          # You can pass in multiple channels to post to by providing a comma-delimited list of channel IDs.
          channel-id: 'xcrawler-github'

          # This data can be any valid JSON from a previous step in the GitHub Action
          # For posting a rich message using Block Kit
          payload: |
            {
              "attachments": [
                {
                  "blocks": [
                    {
                      "type": "header",
                      "text": {
                        "type": "plain_text",
                        "text": "Github Action",
                        "emoji": true
                      }
                    },
                    {
                      "type": "section",
                      "fields": [
                        {
                          "type": "mrkdwn",
                          "text": "*Name:*\n ${{ github.event_name }} "
                        },
                        {
                          "type": "mrkdwn",
                          "text": "*Type:*\n ${{ github.event.action }} ${{ github.event.ref_type }} ${{ github.event.ref }}"
                        }
                      ]
                    },
                    {
                      "type": "section",
                      "fields": [
                        {
                          "type": "mrkdwn",
                          "text": "*Branch:*\n ${{ github.event.pull_request.head.ref }}"
                        },
                        {
                          "type": "mrkdwn",
                          "text": "*Status:*\n `${{ job.status }}`"
                        }
                      ]
                    },
                    {
                      "type": "context",
                      "elements": [
                        {
                          "type": "mrkdwn",
                          "text": "${{ github.event.pull_request.html_url || github.event.head_commit.url }}"
                        }
                      ]
                    }
                  ]
                }
              ]
            }

Microservice – XClient

https://github.com/jooservices/XClient

As a part of refactoring XCrawler. XClient under development

  • Receive crawl request from another service and process request to 3rd party
  • Receive queue request from another service. Queue will be scheduling process later and process callback to another service

Caching also provided

By this way, we don’t need care anything about Curl / Guzzle in another project / service.

Laravel – Different between Model trait boot & initialization

Trace back to Model we have

    protected static function bootTraits()
    {
        $class = static::class;

        $booted = [];

        static::$traitInitializers[$class] = [];

        foreach (class_uses_recursive($class) as $trait) {
            $method = 'boot'.class_basename($trait);

            if (method_exists($class, $method) && ! in_array($method, $booted)) {
                forward_static_call([$class, $method]);

                $booted[] = $method;
            }

            if (method_exists($class, $method = 'initialize'.class_basename($trait))) {
                static::$traitInitializers[$class][] = $method;

                static::$traitInitializers[$class] = array_unique(
                    static::$traitInitializers[$class]
                );
            }
        }
    }

Theo như code trên có thể thấy

  • boot sẽ call đến static method. Hay nói cách khác sẽ tác động tới mọi instances của Model này
  • trong khi đó `initialize thì ngược lại. Chỉ tác động tới instance hiện tại.

1 ví dụ thực tế

trait HasSlug
{
    public function initializeHasSystem()
    {
        $this->mergeCasts([
            'slug' => 'string'
        ]);

        $this->mergeFillable([
            'slug'
        ]);
    }

    public static function bootHasSlug()
    {
        static::creating(function ($model) {
            $model->{$model->getSlugName()} = Str::slug($model->name);
        });

        static::updating(function ($model) {
            $model->{$model->getSlugName()} = Str::slug($model->name);
        });
    }

    public function scopeSlug($query, $uuid)
    {
        return $query->where($this->getSlugName(), $uuid);
    }

    public function getSlugName()
    {
        return property_exists($this, 'slug') ? $this->slug : 'slug';
    }
}

HAProxy Health check fails

backend be_staging
        option httpchk

        server staging 192.168.1.60:80 check

Web server sử dụng Nginx

Đối với config trên thì HAProxy báo be_staging not available dù rằng mọi access đều working fine. !

Check Nginx log ta có

192.168.1.4 - - [27/Aug/2023:21:36:00 +0700] "OPTIONS / HTTP/1.0" 405 166 "-" "-"

Vậy là do Nginx nó trả về 405 cho OPTIONS requests.

Update HAProxy

backend be_staging
        option httpchk GET /
        http-check expect status 200

        server staging 192.168.1.60:80 check

Và mọi thứ trở về lại bình thường !

Laravel – Stable Diffusion

Sau một thời gian nghịch Stable Diffusion thì giờ cũng chút hứng thú với … code Laravel cho em nó.

https://github.com/jooservices/Stable-Diffusion

Về cơ bản

  • Tạo ra queues để liên tục generate images
  • Tạo ra các “base prompt” để có thể tái` sử dụng ( ví dụ có thể tạo ``$prompt->portraits('Irene Aronson') thì sẽ tự add prompt là portraits của nghệ sĩ Irene Aronson

Alrite. Tuy nhiên trước đó thì cần chuẩn bị 1 số thứ

  • Stable Diffusion ( webUI ) chạy service 
Description=systemd service start stable-diffusion

[Service]
Type=simple
Restart=always
RestartSec=1
ExecStart=/bin/bash /home/joos/stable-diffusion-webui/webui.sh
User=joos
WorkingDirectory=/home/joos/stable-diffusion-webui/
StandardOutput=append:/var/log/sdwebui.log
StandardError=append:/var/log/sdwebui.log

[Install]
WantedBy=multi-user.target
  • 1 em cron để chạy schedule:run của Laravel

Xong ! Giờ thì ta có 2 cách

  • Generate ra “queues” qua command
php artisan stable-diffusion:generate --prompt="a beautiful girl at 20 years old. she has colorful hair"

Command này sẽ sinh ra 1.001 queues ứng với prompt trên và … toàn bộ models mình đang có.

  • Code thêm theo ý
        $response = $service
            ->txt2img()
            ->generate($payload)
            ->save($queue->uuid)
            ->getResponse();