Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion content/copilot/reference/ai-models/model-hosting.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ These models are hosted by Amazon Web Services, Anthropic PBC, and Google Cloud
* Anthropic PBC: {% data variables.product.github %} maintains a [zero data retention agreement](https://privacy.anthropic.com/en/articles/8956058-i-have-a-zero-retention-agreement-with-anthropic-what-products-does-it-apply-to) with Anthropic.
* Google Cloud: [Google commits to not training on {% data variables.product.github %} data as part of their service terms](https://cloud.google.com/vertex-ai/generative-ai/docs/data-governance). {% data variables.product.github %} is additionally not subject to prompt logging for abuse monitoring.

To provide better service quality and reduce latency, {% data variables.product.github %} uses [prompt caching](https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching). You can read more about prompt caching on [Anthropic PBC](https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching), [Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html), and [Google Cloud](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude-prompt-caching).
To provide better service quality and reduce latency, {% data variables.product.github %} uses [prompt caching](https://platform.claude.com/docs/en/build-with-claude/prompt-caching). You can read more about prompt caching on [Anthropic PBC](https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching), [Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html), and [Google Cloud](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude-prompt-caching).

When using {% data variables.copilot.copilot_claude %}, input prompts and output completions continue to run through {% data variables.product.prodname_copilot %}'s content filters for public code matching, when applied, along with those for harmful or offensive content.

Expand Down
753 changes: 456 additions & 297 deletions package-lock.json

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@
]
},
"dependencies": {
"@elastic/elasticsearch": "8.13.1",
"@elastic/elasticsearch": "8.19.1",
"@github/failbot": "0.8.3",
"@gr2m/gray-matter": "4.0.3-with-pr-137",
"@horizon-rs/language-guesser": "0.1.1",
Expand Down Expand Up @@ -217,7 +217,7 @@
"mdast-util-to-hast": "^13.2.1",
"mdast-util-to-markdown": "2.1.2",
"mdast-util-to-string": "^4.0.0",
"next": "^16.0.10",
"next": "^16.1.5",
"ora": "^9.0.0",
"parse5": "7.1.2",
"quick-lru": "7.0.1",
Expand Down Expand Up @@ -252,7 +252,7 @@
},
"devDependencies": {
"@actions/core": "^2.0.0",
"@actions/github": "^7.0.0",
"@actions/github": "^8.0.1",
"@axe-core/playwright": "^4.10.1",
"@eslint/js": "^9.33.0",
"@github/markdownlint-github": "^0.6.3",
Expand Down
4 changes: 2 additions & 2 deletions src/search/components/results/SearchResults.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ import styles from './SearchResults.module.scss'

import type { SearchQueryContentT } from '@/search/components/types'
import type { GeneralSearchHitWithoutIncludes, GeneralSearchResponse } from '@/search/types'
import type { SearchTotalHits } from '@elastic/elasticsearch/lib/api/types'
import type { estypes } from '@elastic/elasticsearch'
import { GENERAL_SEARCH_RESULTS } from '@/events/components/event-groups'

type Props = {
results: GeneralSearchResponse
searchParams: SearchQueryContentT
}
export function SearchResults({ results, searchParams }: Props) {
const pages = Math.ceil((results.meta.found as SearchTotalHits).value / results.meta.size)
const pages = Math.ceil((results.meta.found as estypes.SearchTotalHits).value / results.meta.size)
const { page } = results.meta
const searchEventGroupId = useRef<string>('')
useEffect(() => {
Expand Down
4 changes: 2 additions & 2 deletions src/search/components/results/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { NoQuery } from '@/search/components/results/NoQuery'
import { useMainContext } from '@/frame/components/context/MainContext'
import { ValidationErrors } from '@/search/components/results/ValidationErrors'
import { useSearchContext } from '@/search/components/context/SearchContext'
import type { SearchTotalHits } from '@elastic/elasticsearch/lib/api/types'
import type { estypes } from '@elastic/elasticsearch'

export function Search() {
const { search } = useSearchContext()
Expand Down Expand Up @@ -38,7 +38,7 @@ export function Search() {
pageTitle += ` (${searchVersion})`
}
if (results) {
pageTitle = `${formatInteger((results.meta.found as SearchTotalHits).value)} ${pageTitle}`
pageTitle = `${formatInteger((results.meta.found as estypes.SearchTotalHits).value)} ${pageTitle}`
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import type {
AutocompleteMatchQueriesOptions,
AutocompleteResultsArgs,
} from '@/search/lib/get-elasticsearch-results/types'
import type { QueryDslQueryContainer, SearchTotalHits } from '@elastic/elasticsearch/lib/api/types'
import type { estypes } from '@elastic/elasticsearch'

// Query Elasticsearch for AI Search autocomplete results
export async function getAISearchAutocompleteResults({
Expand Down Expand Up @@ -38,7 +38,7 @@ export async function getAISearchAutocompleteResults({
maxLength: 20,
},
})
const matchQuery: QueryDslQueryContainer = {
const matchQuery: estypes.QueryDslQueryContainer = {
bool: {
should: matchQueries,
},
Expand All @@ -65,7 +65,7 @@ export async function getAISearchAutocompleteResults({

return {
meta: {
found: hitsAll.total as SearchTotalHits,
found: hitsAll.total as estypes.SearchTotalHits,
took: { query_msec: result.took, total_msec: new Date().getTime() - t0.getTime() },
size,
},
Expand All @@ -82,7 +82,7 @@ function getAISearchAutocompleteMatchQueries(
const BOOST_PREFIX = 1.0
const BOOST_FUZZY = 0.1

const matchQueries: QueryDslQueryContainer[] = []
const matchQueries: estypes.QueryDslQueryContainer[] = []

// Use match_phrase for exact term matches
matchQueries.push({
Expand Down
19 changes: 7 additions & 12 deletions src/search/lib/get-elasticsearch-results/general-search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,7 @@ import { getElasticsearchClient } from '@/search/lib/helpers/get-client'
import { DEFAULT_HIGHLIGHT_FIELDS } from '@/search/lib/search-request-params/search-params-objects'
import { getHighlightConfiguration } from '@/search/lib/get-elasticsearch-results/helpers/elasticsearch-highlight-config'

import type {
SearchHit as ElasticsearchHit,
QueryDslQueryContainer,
SearchRequest,
SearchTotalHits,
} from '@elastic/elasticsearch/lib/api/types'
import type { estypes } from '@elastic/elasticsearch'
import type {
AdditionalIncludes,
ComputedSearchQueryParamsMap,
Expand Down Expand Up @@ -122,7 +117,7 @@ export async function getGeneralSearchResults(

const aggs = getAggregations(aggregate)

const searchQuery: SearchRequest = {
const searchQuery: estypes.SearchRequest = {
index: indexName,
highlight,
from,
Expand Down Expand Up @@ -198,7 +193,7 @@ export async function getGeneralSearchResults(
const t1 = Date.now()

const meta = {
found: hitsAll.total as SearchTotalHits,
found: hitsAll.total as estypes.SearchTotalHits,
took: {
query_msec: result.took,
total_msec: t1 - t0,
Expand Down Expand Up @@ -255,7 +250,7 @@ interface GetMatchQueriesOptions {
function getMatchQueries(
query: string,
{ usePrefixSearch, fuzzy }: GetMatchQueriesOptions,
): QueryDslQueryContainer[] {
): estypes.QueryDslQueryContainer[] {
const BOOST_PHRASE = 10.0
const BOOST_TITLE = 4.0
const BOOST_HEADINGS = 3.0
Expand All @@ -268,7 +263,7 @@ function getMatchQueries(
// which wouldn't find anything else anyway.
const BOOST_FUZZY = 0.1

const matchQueries: QueryDslQueryContainer[] = []
const matchQueries: estypes.QueryDslQueryContainer[] = []

// If the query input is multiple words, it's good to know because you can
// make the query do `match_phrase` and you can make `match` query
Expand Down Expand Up @@ -452,7 +447,7 @@ interface GetHitsOptions {
}

function getHits(
hits: ElasticsearchHit<any>[],
hits: estypes.SearchHit<any>[],
{ indexName, debug = false, includeTopics = false, highlightFields, include }: GetHitsOptions,
): GeneralSearchHit[] {
return hits.map((hit) => {
Expand All @@ -470,7 +465,7 @@ function getHits(
}

const result: GeneralSearchHit = {
id: hit._id,
id: hit._id!,
url: hit._source.url,
title: hit._source.title,
breadcrumbs: hit._source.breadcrumbs,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { SearchHighlight } from '@elastic/elasticsearch/lib/api/types'
import type { estypes } from '@elastic/elasticsearch'

import type { HighlightOptions } from '@/search/lib/search-request-params/types'

Expand All @@ -18,7 +18,7 @@ export type HighlightFields = {
export function getHighlightConfiguration(
query: string,
highlightsFields: HighlightOptions[],
): SearchHighlight {
): estypes.SearchHighlight {
const fields = {} as HighlightFields
if (highlightsFields.includes('title')) {
fields.title = {
Expand Down Expand Up @@ -76,7 +76,7 @@ export function getHighlightConfiguration(
}
}

const highlightConfig: SearchHighlight = {
const highlightConfig: estypes.SearchHighlight = {
pre_tags: ['<mark>'],
post_tags: ['</mark>'],
fields,
Expand Down
4 changes: 2 additions & 2 deletions src/search/scripts/analyze-text.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import dotenv from 'dotenv'
import { languageKeys } from '@/languages/lib/languages-server'
import { allVersions } from '@/versions/lib/all-versions'

import type { IndicesAnalyzeAnalyzeToken } from '@elastic/elasticsearch/lib/api/types'
import type { estypes } from '@elastic/elasticsearch'

// Now you can optionally have set the ELASTICSEARCH_URL in your .env file.
dotenv.config()
Expand Down Expand Up @@ -157,7 +157,7 @@ async function analyzeVersion(client: Client, texts: string[], indexName: string
body: { analyzer, text },
})

const tokens: IndicesAnalyzeAnalyzeToken[] | undefined = response.tokens
const tokens: estypes.IndicesAnalyzeAnalyzeToken[] | undefined = response.tokens
const tokenWords: string[] = tokens?.map((token) => token.token) || []
console.log(tokenWords)
}
Expand Down
24 changes: 11 additions & 13 deletions src/search/scripts/index/utils/settings.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
import { SNOWBALL_LANGUAGES } from '@/search/scripts/index/utils/constants'

import type { estypes } from '@elastic/elasticsearch'
import type {
AnalysisSnowballLanguage,
AnalysisCustomAnalyzer,
} from '@elastic/elasticsearch/lib/api/types'

export function getGeneralSearchSettings(
language: string,
Expand All @@ -24,25 +20,26 @@ export function getGeneralSearchSettings(
filter: ['lowercase', 'stop', 'asciifolding'],
tokenizer: 'standard',
type: 'custom',
} as AnalysisCustomAnalyzer,
} as estypes.AnalysisCustomAnalyzer,
text_analyzer: {
filter: ['lowercase', 'stop', 'asciifolding'],
tokenizer: 'standard',
type: 'custom',
} as AnalysisCustomAnalyzer,
} as estypes.AnalysisCustomAnalyzer,
},
filter: {},
},
}

const snowballLanguage = SNOWBALL_LANGUAGES[language]
if (snowballLanguage) {
const textAnalyzer = settings.analysis!.analyzer!.text_analyzer as AnalysisCustomAnalyzer
textAnalyzer.filter!.push('languaged_snowball')
const textAnalyzer = settings.analysis!.analyzer!
.text_analyzer as estypes.AnalysisCustomAnalyzer
;(textAnalyzer.filter as string[]).push('languaged_snowball')

settings.analysis!.filter!['languaged_snowball'] = {
type: 'snowball',
language: snowballLanguage as AnalysisSnowballLanguage,
language: snowballLanguage as estypes.AnalysisSnowballLanguage,
}
} else if (verbose) {
console.warn(`No snowball language for '${language}'`)
Expand All @@ -62,20 +59,21 @@ export function getAISearchAutocompleteSettings(
filter: ['lowercase'],
tokenizer: 'standard',
type: 'custom',
} as AnalysisCustomAnalyzer,
} as estypes.AnalysisCustomAnalyzer,
},
filter: {},
},
}

const snowballLanguage = SNOWBALL_LANGUAGES[language]
if (snowballLanguage) {
const textAnalyzer = settings.analysis!.analyzer!.text_analyzer as AnalysisCustomAnalyzer
textAnalyzer.filter!.push('languaged_snowball')
const textAnalyzer = settings.analysis!.analyzer!
.text_analyzer as estypes.AnalysisCustomAnalyzer
;(textAnalyzer.filter as string[]).push('languaged_snowball')

settings.analysis!.filter!['languaged_snowball'] = {
type: 'snowball',
language: snowballLanguage as AnalysisSnowballLanguage,
language: snowballLanguage as estypes.AnalysisSnowballLanguage,
}
} else if (verbose) {
console.warn(`No snowball language for '${language}'`)
Expand Down
4 changes: 2 additions & 2 deletions src/search/types.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { SearchTotalHits } from '@elastic/elasticsearch/lib/api/types'
import type { estypes } from '@elastic/elasticsearch'

import type {
AdditionalIncludes,
Expand Down Expand Up @@ -73,7 +73,7 @@ export type SearchResultAggregations = {
}

type SearchResultsMeta = {
found: SearchTotalHits
found: estypes.SearchTotalHits
took: {
query_msec: number
total_msec: number
Expand Down
Loading