AWS

CloudWatch CLI

aws logs filter-log-events --log-group-name /aws/lambda/LogGroupName --output json --region ap-southeast-1 --start-time 1706918400000 --end-time 1707004800000 --query "events[?contains(message, 'some text')].{Timestamp:timestamp, Message:message}" > result.json

CloudWatch Logs Insights

Case-insensitive search

sql

fields @timestamp, @message, @logStream, @log
| filter @message like /(?i)(something)/
| sort @timestamp desc
| limit 10000

Tail Lambda logs

aws logs tail /aws/lambda/$2 \
  --follow \
  --region ap-southeast-1 \
  --profile $1 \
  --format short

Lambda

Update Lambda code

aws lambda update-function-code  --region ap-southeast-1  --profile my-profile --function-name YourFunctionName  --zip-file fileb://path/to/your/file.zip

aws lambda get-function-configuration --function-name YourFunctionName --region ap-southeast-1 --profile my-profile | jq ."Environment.Variables"

aws lambda update-function-configuration --region ap-southeast-1 --function-name my-lambda-function --environment "Variables={VAR1=new_value1,VAR2=new_value2}"

aws lambda update-function-configuration --region ap-southeast-1 --function-name function-name --environment "Variables={$(cat env.config | jq -r 'to_entries | map("\(.key)=\(.value | gsub(","; "\\,"))") | join(",")')}"

Lambda starter kit

pnpm add -D esbuild eslint typescript @typescript-eslint/eslint-plugin @typescript-eslint/parser @types/node @types/aws-lambda globals typescript-eslint @eslint/js
pnpm add @aws-sdk/client-lambda

json

{
  "build": "rm -rf dist && esbuild src/index.ts --bundle --minify --sourcemap --platform=node --target=es2022 --outfile=dist/index.js",
}

DynamoDB

@aws-sdk/lib-dynamodb handles marshalling and unmarshalling. @aws-sdk/client-dynamodb returns the raw DynamoDB format.

Template

import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
import {
  DynamoDBDocumentClient,
  ScanCommand,          // <-- from @aws-sdk/lib-dynamodb
} from "@aws-sdk/lib-dynamodb";

const ddbClient = new DynamoDBClient({});

const marshallOptions = {
  convertEmptyValues: false,
  removeUndefinedValues: true,
  convertClassInstanceToMap: false,
};
const unmarshallOptions = { wrapNumbers: false };
const translateConfig = { marshallOptions, unmarshallOptions };

const ddbDocClient = DynamoDBDocumentClient.from(ddbClient, translateConfig);

// Use plain JS in your params; no {S:".."} etc.
const params = { TableName: "MyTable" };

const data = await ddbDocClient.send(new ScanCommand(params));
console.log(data.Items); // => [{ id: "123", count: 5, ... }]  // normal JSON

Note: Import the command from @aws-sdk/lib-dynamodb instead of @aws-sdk/client-dynamodb, then send it with DynamoDBDocumentClient.

Get items

const params = {
  TableName: `some_table`,
  Key: {
    id: partitionKey,
    timestamp: sortKey
  },
};
const data = await ddbClient.send(new GetCommand(params));

Add an item

const putParams = {
  TableName: `table_name`,
  Item: {
    id: some_id,
    createTime: now.getTime(),
    status: 'P',
    payer,
    details,
    amount
  },
};
await ddbDocClient.send(new PutCommand(putParams)).catch((err) => {
  log.error(err);
});

Update an item

const params = {
  TableName: `some_table`,
  Key: {
    id: 'some_id',
  },
  Item: someObject,
  ExpressionAttributeNames: { '#v': 'value', },
  UpdateExpression: 'set #v = :v',
  ExpressionAttributeValues: {
    ':v': someObject 
  },
};
await ddbDocClient.send(new UpdateCommand(params)).catch((err) => {
  // error handling
}

Delete a field

const params = {
  TableName: `some_table`,
  Key: {
    id: 'some_id',
  },
  UpdateExpression: 'REMOVE some_field',
};
await ddbDocClient.send(new UpdateCommand(params)).catch(err => {
  // error handling
});

Use transactions

const params: TransactWriteCommandInput = {
  TransactItems: [{
    Update: {
      ... // some transaction
    }
  }, {
    Put: {
      ... // some transaction
    }
  }]
};
await ddbDocClient.send(new TransactWriteCommand(params)).catch(async (err) => {
  // error handling
});

Scan

Supported functions:

contains
begins_with
attribute_exists
attribute_not_exists
attribute_type
size

Supported logical operations:

AND
OR
NOT

Supported operators:

Operator	Meaning
`=`	equal
`<>`	not equal
`<` `<=` `>` `>=`	comparisons
`BETWEEN`	range
`IN`	value in list

const params: UpdateParam = {
  TableName: `some_table`,
  ExpressionAttributeNames: { '#d': 'date' },
  ExpressionAttributeValues: {
    ':d1': from_date,
    ':d2': to_date
  },
  ScanIndexForward: false,
  FilterExpression: '#d BETWEEN :d1 and :d2'  // other function: contains
};
try {
  let data;
  do {
    if (data) {
      params.ExclusiveStartKey = data.LastEvaluatedKey;
    }

    data = await ddbClient.send(new ScanCommand(params));
    if (data.Items && data.Items instanceof Array) {
      for (const item of data.Items) {
        // append to list
      }
    }
  } while (data.LastEvaluatedKey);

Append to a list

const params = {
  TableName: `some_table`,
  Key: {
    id: some_key
  },
  ExpressionAttributeNames: {
    '#l': 'list'
  },
  ExpressionAttributeValues: {
    ':l': arrayList,
    ':emptyList': []
  },
  UpdateExpression: 'SET #l = list_append(if_not_exists(#l, :emptyList), :l)'
}

Update only if the row exists

const params = {
  TableName: `some_table`,
  Key: {
    id: some_key.
    sid: some_sort_key
  },
  ExpressionAttributeNames: { '#v': 'value', },
  UpdateExpression: 'set #v = :v',
  ExpressionAttributeValues: {
    ':v': someObject 
  },
  ConditionExpression: 'attribute_exists(#id) AND attribute_exists(#sid)'
}

Update only if the row does not exist

const params = {
  TableName: `some_table`,
  Key: {
    id: some_key.
    sid: some_sort_key
  },
  ExpressionAttributeNames: { '#v': 'value', },
  UpdateExpression: 'set #v = :v',
  ExpressionAttributeValues: {
    ':v': someObject 
  },
  ConditionExpression: 'attribute_not_exists(#id) AND attribute_not_exists(#sid)'
}

Batch get

const params: BatchGetCommandInput = {
  RequestItems: {
    [tableName]: {
      Keys: [ ... some_keys ]
    }
  }
};
const data = await ddbClient.send(new BatchGetCommand(params));
if (data.Responses && data.Responses[tableName]) {
  return data.Responses[tableName];
}

Batch write

const params: BatchWriteCommandInput = {
  RequestItems: {
    [`table_name`]: [
      PutRequest: {
        Item: {
          id: item.id,
          value: item.value
        }
      }
    ]
  }
};
await ddbDocClient.send(new BatchWriteCommand(params)).catch((err) => {
  // Error handling
});

Delete a row

const params: DeleteCommandInput = {
  TableName: 'table_name',
  Key: {
    id: some_id 
  }
};
await ddbClient.send(new DeleteCommand(params)).catch((error) => {
  // Error handling
});

EC2 starter kit

For the Ubuntu EC2 bootstrap script, refer to Operating System.

S3

Download a folder. Add --dryrun for preview

aws s3 sync s3://bucket_name/folder/ ./target/ --region ap-southeast-1 --profile profile_name

Cloud front deployment script

This script will delete the file in s3 and re-upload files. It will ignore all the hidden file (file start with .)

#!/bin/bash

set -e

BUCKET_NAME="$1"
CLOUDFRONT_DISTRIBUTION_ID="$2"
AWS_PROFILE="$3"
AWS_REGION="$4"

if [ -z "$BUCKET_NAME" ] || [ -z "$CLOUDFRONT_DISTRIBUTION_ID" ]; then
  echo "Usage: $0 <bucket-name> <cloudfront-distribution-id> [aws-profile] [aws-region]"
  exit 1
fi

PROFILE_ARG=""
REGION_ARG=""

if [ -n "$AWS_PROFILE" ]; then
  PROFILE_ARG="--profile $AWS_PROFILE"
fi

if [ -n "$AWS_REGION" ]; then
  REGION_ARG="--region $AWS_REGION"
fi

echo "Syncing ./dist to s3://$BUCKET_NAME ..."

aws s3 sync ./dist "s3://$BUCKET_NAME" \
  --delete \
  --exclude ".*" \
  --exclude "*/.*" \
  $PROFILE_ARG \
  $REGION_ARG

echo "Invalidating CloudFront distribution: $CLOUDFRONT_DISTRIBUTION_ID ..."

aws cloudfront create-invalidation \
  --distribution-id "$CLOUDFRONT_DISTRIBUTION_ID" \
  --paths "/*" \
  $PROFILE_ARG

echo "Deployment completed."

Mark obsolete

This script will compare the file in ./dist folder with s3 and tag status of the file to obsolete if not exists.

#!/bin/bash

set -euo pipefail

BUCKET_NAME="$1"
AWS_PROFILE="${2:-}"
AWS_REGION="${3:-}"
S3_PREFIX="${4:-}"

DIST_DIR="./dist"

if [ -z "$BUCKET_NAME" ]; then
  echo "Usage: $0 <bucket-name> [aws-profile] [aws-region] [s3-prefix]"
  exit 1
fi

if [ ! -d "$DIST_DIR" ]; then
  echo "Error: $DIST_DIR does not exist."
  exit 1
fi

if ! command -v jq >/dev/null 2>&1; then
  echo "Error: jq is required."
  exit 1
fi

PROFILE_ARGS=()
REGION_ARGS=()

if [ -n "$AWS_PROFILE" ]; then
  PROFILE_ARGS=(--profile "$AWS_PROFILE")
fi

if [ -n "$AWS_REGION" ]; then
  REGION_ARGS=(--region "$AWS_REGION")
fi

# Normalize prefix
if [ -n "$S3_PREFIX" ]; then
  S3_PREFIX="${S3_PREFIX#/}"
  S3_PREFIX="${S3_PREFIX%/}/"
fi

S3_FILES=$(mktemp)
LOCAL_FILES=$(mktemp)
OBSOLETE_FILES=$(mktemp)

trap 'rm -f "$S3_FILES" "$LOCAL_FILES" "$OBSOLETE_FILES"' EXIT

echo "Getting S3 file list..."

aws s3api list-objects-v2 \
  --bucket "$BUCKET_NAME" \
  ${S3_PREFIX:+--prefix "$S3_PREFIX"} \
  "${PROFILE_ARGS[@]}" \
  "${REGION_ARGS[@]}" \
  --output json \
  | jq -r '.Contents[]?.Key' \
  | sort > "$S3_FILES"

echo "Getting local ./dist file list..."

find "$DIST_DIR" -type f \
  ! -name ".DS_Store" \
  ! -name "._*" \
  ! -path "*/.*" \
  | sed "s|^${DIST_DIR}/||" \
  | sed "s|^|${S3_PREFIX}|" \
  | sort > "$LOCAL_FILES"

echo "Comparing files..."

comm -23 "$S3_FILES" "$LOCAL_FILES" > "$OBSOLETE_FILES"

OBSOLETE_COUNT=$(wc -l < "$OBSOLETE_FILES" | tr -d ' ')

echo "Found $OBSOLETE_COUNT obsolete file(s)."

if [ "$OBSOLETE_COUNT" -eq 0 ]; then
  echo "Nothing to tag."
  exit 0
fi

while IFS= read -r KEY; do
  [ -z "$KEY" ] && continue

  echo "Tagging obsolete: $KEY"

  # Get existing tags
  EXISTING_TAGS=$(
    aws s3api get-object-tagging \
      --bucket "$BUCKET_NAME" \
      --key "$KEY" \
      "${PROFILE_ARGS[@]}" \
      "${REGION_ARGS[@]}" \
      --output json
  )

  # Preserve existing tags, remove old "status" tag if present,
  # then add status=obsolete
  NEW_TAGS=$(
    echo "$EXISTING_TAGS" | jq '
      {
        TagSet:
          (
            [.TagSet[]? | select(.Key != "status")]
            + [{"Key": "status", "Value": "obsolete"}]
          )
      }
    '
  )

  aws s3api put-object-tagging \
    --bucket "$BUCKET_NAME" \
    --key "$KEY" \
    --tagging "$NEW_TAGS" \
    "${PROFILE_ARGS[@]}" \
    "${REGION_ARGS[@]}"

done < "$OBSOLETE_FILES"

echo "Completed."
echo "$OBSOLETE_COUNT file(s) tagged with status=obsolete."

Floci

AWS, GCP, Azure simulator

Floci core

yml

services:
  floci:
    image: floci/floci:latest
    ports:
      - "4566:4566"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./data:/app/data
    environment:
      FLOCI_STORAGE_MODE: hybrid

Floci ui

yml

services:
  floci-ui:
    build:
      context: ./packages/frontend
      dockerfile: Dockerfile.dev
    volumes:
      - ./packages/frontend/src:/app/src
      - ./packages/frontend/index.html:/app/index.html
      - ./packages/frontend/vite.config.ts:/app/vite.config.ts
      - ./packages/frontend/tsconfig.json:/app/tsconfig.json
      - ./packages/frontend/tsconfig.node.json:/app/tsconfig.node.json
    ports:
      - "4500:4500"
    environment:
      API_TARGET: http://floci-api:4501
      VITE_USE_POLLING: "true"
    depends_on:
      - floci-api
    networks:
      - floci_default

  floci-api:
    build:
      context: ./packages/api
      dockerfile: Dockerfile.dev
    volumes:
      - ./packages/api/src:/app/src
    ports:
      - "4501:4501"
    environment:
      FLOCI_ENDPOINT: http://floci:4566
      FLOCI_AZURE_ENDPOINT: http://floci-az:4577
      AWS_REGION: us-east-1
      AWS_ACCESS_KEY_ID: test
      AWS_SECRET_ACCESS_KEY: test
      FLOCI_GCP_ENDPOINT: http://floci-gcp:4588
      FLOCI_GCP_PROJECT: floci-local
      PORT: "4501"
    depends_on:
      - floci
    networks:
      - floci_default

  floci:
    image: floci/floci:latest-compat
    ports:
      - "4566:4566"
    volumes:
      # Floci runs Lambda functions in throwaway containers, so it needs the host
      # Docker daemon. Without this, invoke fails with "Failed to start Lambda
      # container". This grants the emulator control of host Docker — acceptable
      # for a local dev stack, and the same mount the README's docker run uses.
      - /var/run/docker.sock:/var/run/docker.sock
      - ./data:/app/data
      - ./init/ready.d:/etc/floci/init/ready.d:ro
      - ./init/files:/etc/floci/init/files:ro
    environment:
      FLOCI_STORAGE_MODE: persistent
    networks:
      floci_default:
        aliases:
          - localhost.floci.io

  # Multi-cloud emulators — enabled with the "multicloud" profile:
  #   docker compose --profile multicloud up
  floci-az:
    image: floci/floci-az
    profiles: ["multicloud"]
    ports:
      - "4577:4577"
    networks:
      - floci_default

  floci-gcp:
    image: floci/floci-gcp
    profiles: ["multicloud"]
    ports:
      - "4588:4588"
    volumes:
      # Cloud SQL instances are real Postgres containers; see the note on `floci`.
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - floci_default

  floci-seed:
    image: floci/floci-gcp
    profiles: ["multicloud"]
    command: ["/bin/sh", "/etc/floci/init/seed-clouds.sh"]
    volumes:
      - ./init:/etc/floci/init:ro
    environment:
      FLOCI_AZURE_ENDPOINT: http://floci-az:4577
      FLOCI_AZURE_ACCOUNT_NAME: devstoreaccount1
      FLOCI_GCP_ENDPOINT: http://floci-gcp:4588
      FLOCI_GCP_PROJECT: floci-local
    depends_on:
      - floci-az
      - floci-gcp
    networks:
      - floci_default

networks:
  floci_default:
    name: floci_default

AWS ​

CloudWatch CLI ​

CloudWatch Logs Insights ​

Case-insensitive search ​

Tail Lambda logs ​

Lambda ​

Update Lambda code ​

Lambda starter kit ​

DynamoDB ​

Template ​

Get items ​

Add an item ​

Update an item ​

Delete a field ​

Use transactions ​

Scan ​

Append to a list ​

Update only if the row exists ​

Update only if the row does not exist ​

Batch get ​

Batch write ​

Delete a row ​

EC2 starter kit ​

S3 ​

Cloud front deployment script ​

Mark obsolete ​

Floci ​

Floci core ​

Floci ui ​

AWS